You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/11/18 15:29:35 UTC
[01/28] carbondata git commit:
[CARBONDATA-1612][CARBONDATA-1615][Streaming] Support delete segment for
streaming table [Forced Update!]
Repository: carbondata
Updated Branches:
refs/heads/fgdatamap 40259b362 -> 36c34cd02 (forced update)
[CARBONDATA-1612][CARBONDATA-1615][Streaming] Support delete segment for streaming table
This closes #1497
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/09d02056
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/09d02056
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/09d02056
Branch: refs/heads/fgdatamap
Commit: 09d020561a6a5b7bd90b769e608b4130baa43667
Parents: 1155d4d
Author: Jacky Li <ja...@qq.com>
Authored: Tue Nov 14 20:41:24 2017 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Wed Nov 15 09:35:53 2017 +0800
----------------------------------------------------------------------
.../spark/sql/CarbonCatalystOperators.scala | 4 +-
.../TestStreamingTableOperation.scala | 55 +++++++++++++++++++-
2 files changed, 55 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/09d02056/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
index 48f1a09..62632df 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
@@ -119,10 +119,10 @@ case class ShowLoadsCommand(
extends Command {
override def output: Seq[Attribute] = {
- Seq(AttributeReference("SegmentSequenceId", StringType, nullable = false)(),
+ Seq(AttributeReference("Segment Id", StringType, nullable = false)(),
AttributeReference("Status", StringType, nullable = false)(),
AttributeReference("Load Start Time", TimestampType, nullable = false)(),
- AttributeReference("Load End Time", TimestampType, nullable = false)(),
+ AttributeReference("Load End Time", TimestampType, nullable = true)(),
AttributeReference("Merged To", StringType, nullable = false)())
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/09d02056/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 3fb1424..b29cca4 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -19,6 +19,7 @@ package org.apache.spark.carbondata
import java.io.{File, PrintWriter}
import java.net.ServerSocket
+import java.util.{Calendar, Date}
import java.util.concurrent.Executors
import scala.collection.mutable
@@ -103,6 +104,9 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
// 10. fault tolerant
createTable(tableName = "stream_table_tolerant", streaming = true, withBatchLoad = true)
+
+ // 11. table for delete segment test
+ createTable(tableName = "stream_table_delete", streaming = true, withBatchLoad = false)
}
test("validate streaming property") {
@@ -181,6 +185,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists streaming.stream_table_compact")
sql("drop table if exists streaming.stream_table_new")
sql("drop table if exists streaming.stream_table_tolerant")
+ sql("drop table if exists streaming.stream_table_delete")
}
// normal table not support streaming ingest
@@ -578,8 +583,6 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
badRecordAction = "force",
handoffSize = 1024L * 200
)
- sql("show segments for table streaming.stream_table_new").show(100, false)
-
assert(sql("show segments for table streaming.stream_table_new").count() == 4)
checkAnswer(
@@ -588,6 +591,51 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test deleting streaming segment by ID while ingesting") {
+ executeStreamingIngest(
+ tableName = "stream_table_delete",
+ batchNums = 6,
+ rowNumsEachBatch = 10000,
+ intervalOfSource = 3,
+ intervalOfIngest = 5,
+ continueSeconds = 15,
+ generateBadRecords = false,
+ badRecordAction = "force",
+ handoffSize = 1024L * 200
+ )
+ val beforeDelete = sql("show segments for table streaming.stream_table_delete").collect()
+ val segmentId = beforeDelete.map(_.getString(0)).mkString(",")
+ sql(s"delete from table streaming.stream_table_delete where segment.id in ($segmentId) ")
+
+ val rows = sql("show segments for table streaming.stream_table_delete").collect()
+ rows.foreach { row =>
+ assertResult(SegmentStatus.MARKED_FOR_DELETE.getMessage)(row.getString(1))
+ }
+ }
+
+ test("test deleting streaming segment by date while ingesting") {
+ executeStreamingIngest(
+ tableName = "stream_table_delete",
+ batchNums = 6,
+ rowNumsEachBatch = 10000,
+ intervalOfSource = 3,
+ intervalOfIngest = 5,
+ continueSeconds = 15,
+ generateBadRecords = false,
+ badRecordAction = "force",
+ handoffSize = 1024L * 200
+ )
+ val beforeDelete = sql("show segments for table streaming.stream_table_delete").collect()
+
+ sql(s"delete from table streaming.stream_table_delete where segment.starttime before '2999-10-01 01:00:00'")
+
+ val rows = sql("show segments for table streaming.stream_table_delete").collect()
+ assertResult(beforeDelete.length)(rows.length)
+ rows.foreach { row =>
+ assertResult(SegmentStatus.MARKED_FOR_DELETE.getMessage)(row.getString(1))
+ }
+ }
+
def createWriteSocketThread(
serverSocket: ServerSocket,
writeNums: Int,
@@ -675,6 +723,9 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
}
}
+ /**
+ * start ingestion thread: write `rowNumsEachBatch` rows repeatly for `batchNums` times.
+ */
def executeStreamingIngest(
tableName: String,
batchNums: Int,
[10/28] carbondata git commit: [CARBONDATA-1739] Clean up store path
interface
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
index 822455c..64a066c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
@@ -47,9 +47,7 @@ case class CarbonDataMapShowCommand(
override def processData(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
- lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
val schemaList = carbonTable.getTableInfo.getDataMapSchemaList
if (schemaList != null && schemaList.size() > 0) {
schemaList.asScala.map { s =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 66f2756..f34afbf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -34,6 +34,7 @@ import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events._
@@ -60,12 +61,11 @@ case class CarbonDropDataMapCommand(
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
val dbName = GetDB.getDatabaseName(databaseNameOp, sparkSession)
val identifier = TableIdentifier(tableName, Option(dbName))
- val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
val locksToBeAcquired = List(LockUsage.METADATA_LOCK)
val carbonEnv = CarbonEnv.getInstance(sparkSession)
val catalog = carbonEnv.carbonMetastore
val databaseLocation = GetDB.getDatabaseLocation(dbName, sparkSession,
- CarbonEnv.getInstance(sparkSession).storePath)
+ CarbonProperties.getStorePath)
val tablePath = databaseLocation + CarbonCommonConstants.FILE_SEPARATOR + tableName.toLowerCase
val tableIdentifier =
AbsoluteTableIdentifier.from(tablePath, dbName.toLowerCase, tableName.toLowerCase)
@@ -76,20 +76,19 @@ case class CarbonDropDataMapCommand(
lock => carbonLocks += CarbonLockUtil.getLockObject(tableIdentifier, lock)
}
LOGGER.audit(s"Deleting datamap [$dataMapName] under table [$tableName]")
- val carbonTable: Option[CarbonTable] =
- catalog.getTableFromMetadataCache(dbName, tableName) match {
- case Some(tableMeta) => Some(tableMeta.carbonTable)
- case None => try {
- Some(catalog.lookupRelation(identifier)(sparkSession)
- .asInstanceOf[CarbonRelation].metaData.carbonTable)
- } catch {
- case ex: NoSuchTableException =>
- if (!ifExistsSet) {
- throw ex
- }
- None
- }
+ var carbonTable: Option[CarbonTable] =
+ catalog.getTableFromMetadataCache(dbName, tableName)
+ if (carbonTable.isEmpty) {
+ try {
+ carbonTable = Some(catalog.lookupRelation(identifier)(sparkSession)
+ .asInstanceOf[CarbonRelation].metaData.carbonTable)
+ } catch {
+ case ex: NoSuchTableException =>
+ if (!ifExistsSet) {
+ throw ex
+ }
}
+ }
if (carbonTable.isDefined && carbonTable.get.getTableInfo.getDataMapSchemaList.size() > 0) {
val dataMapSchema = carbonTable.get.getTableInfo.getDataMapSchemaList.asScala.zipWithIndex.
find(_._1.getDataMapName.equalsIgnoreCase(dataMapName))
@@ -144,7 +143,7 @@ case class CarbonDropDataMapCommand(
// delete the table folder
val dbName = GetDB.getDatabaseName(databaseNameOp, sparkSession)
val databaseLocation = GetDB.getDatabaseLocation(dbName, sparkSession,
- CarbonEnv.getInstance(sparkSession).storePath)
+ CarbonProperties.getStorePath)
val tablePath = databaseLocation + CarbonCommonConstants.FILE_SEPARATOR + tableName.toLowerCase
val tableIdentifier = AbsoluteTableIdentifier.from(tablePath, dbName, tableName)
DataMapStoreManager.getInstance().clearDataMap(tableIdentifier, dataMapName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala
index 947cea1..2f04feb 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala
@@ -57,21 +57,21 @@ case class AlterTableCompactionCommand(
if (relation == null) {
sys.error(s"Table $databaseName.$tableName does not exist")
}
- if (null == relation.tableMeta.carbonTable) {
+ if (null == relation.carbonTable) {
LOGGER.error(s"alter table failed. table not found: $databaseName.$tableName")
sys.error(s"alter table failed. table not found: $databaseName.$tableName")
}
val carbonLoadModel = new CarbonLoadModel()
- val table = relation.tableMeta.carbonTable
- carbonLoadModel.setTableName(table.getFactTableName)
+ val table = relation.carbonTable
+ carbonLoadModel.setTableName(table.getTableName)
val dataLoadSchema = new CarbonDataLoadSchema(table)
// Need to fill dimension relation
carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setTablePath(relation.tableMeta.carbonTable.getTablePath)
+ carbonLoadModel.setTableName(relation.carbonTable.getTableName)
+ carbonLoadModel.setDatabaseName(relation.carbonTable.getDatabaseName)
+ carbonLoadModel.setTablePath(relation.carbonTable.getTablePath)
var storeLocation = CarbonProperties.getInstance
.getProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala
index 2003bb1..32d6b80 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala
@@ -37,9 +37,7 @@ case class CarbonShowLoadsCommand(
override def processData(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
- lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
CarbonStore.showSegments(
GetDB.getDatabaseName(databaseNameOp, sparkSession),
tableName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CleanFilesCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CleanFilesCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CleanFilesCommand.scala
index 8b0dab7..58e33b7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CleanFilesCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CleanFilesCommand.scala
@@ -22,6 +22,7 @@ import org.apache.spark.sql.execution.command.{Checker, DataProcessCommand, Runn
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.api.CarbonStore
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.{CleanFilesPostEvent, CleanFilesPreEvent, OperationContext, OperationListenerBus}
case class CleanFilesCommand(
@@ -38,7 +39,7 @@ case class CleanFilesCommand(
if (forceTableClean) {
val dbName = GetDB.getDatabaseName(databaseNameOp, sparkSession)
val databaseLocation = GetDB.getDatabaseLocation(dbName, sparkSession,
- CarbonEnv.getInstance(sparkSession).storePath)
+ CarbonProperties.getStorePath)
// TODO: TAABLEPATH
CarbonStore.cleanFiles(
dbName,
@@ -47,10 +48,7 @@ case class CleanFilesCommand(
null,
forceTableClean)
} else {
- val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val relation = catalog
- .lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation]
- val carbonTable = relation.tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
val cleanFilesPreEvent: CleanFilesPreEvent =
CleanFilesPreEvent(carbonTable,
sparkSession)
@@ -59,7 +57,7 @@ case class CleanFilesCommand(
CarbonStore.cleanFiles(
GetDB.getDatabaseName(databaseNameOp, sparkSession),
tableName,
- relation.asInstanceOf[CarbonRelation].tableMeta.storePath,
+ CarbonProperties.getStorePath,
carbonTable,
forceTableClean)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByIdCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByIdCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByIdCommand.scala
index 6a0465c..5b305ba 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByIdCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByIdCommand.scala
@@ -35,9 +35,7 @@ case class DeleteLoadByIdCommand(
override def processData(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
- lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
val operationContext = new OperationContext
val deleteSegmentByIdPreEvent: DeleteSegmentByIdPreEvent =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByLoadDateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByLoadDateCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByLoadDateCommand.scala
index 83f41bb..00c35a5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByLoadDateCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/DeleteLoadByLoadDateCommand.scala
@@ -37,9 +37,7 @@ case class DeleteLoadByLoadDateCommand(
override def processData(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
- lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
val operationContext = new OperationContext
val deleteSegmentByDatePreEvent: DeleteSegmentByDatePreEvent =
DeleteSegmentByDatePreEvent(carbonTable,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableByInsertCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableByInsertCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableByInsertCommand.scala
index 3f0e093..845a64c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableByInsertCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableByInsertCommand.scala
@@ -47,7 +47,7 @@ case class LoadTableByInsertCommand(
Some(df)).run(sparkSession)
// updating relation metadata. This is in case of auto detect high cardinality
relation.carbonRelation.metaData =
- CarbonSparkUtil.createSparkMeta(relation.carbonRelation.tableMeta.carbonTable)
+ CarbonSparkUtil.createSparkMeta(relation.carbonRelation.carbonTable)
load
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableCommand.scala
index 777c169..0f4ca01 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/LoadTableCommand.scala
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOp
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.metadata.encoder.Encoding
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
import org.apache.carbondata.core.statusmanager.SegmentStatus
@@ -54,7 +55,8 @@ case class LoadTableCommand(
isOverwriteTable: Boolean,
var inputSqlString: String = null,
dataFrame: Option[DataFrame] = None,
- updateModel: Option[UpdateTableModel] = None)
+ updateModel: Option[UpdateTableModel] = None,
+ var tableInfoOp: Option[TableInfo] = None)
extends RunnableCommand with DataProcessCommand {
override def run(sparkSession: SparkSession): Seq[Row] = {
@@ -72,16 +74,6 @@ case class LoadTableCommand(
}
val dbName = databaseNameOp.getOrElse(sparkSession.catalog.currentDatabase)
- val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
- if (relation == null) {
- sys.error(s"Table $dbName.$tableName does not exist")
- }
- if (null == relation.tableMeta.carbonTable) {
- LOGGER.error(s"Data loading failed. table not found: $dbName.$tableName")
- LOGGER.audit(s"Data loading failed. table not found: $dbName.$tableName")
- sys.error(s"Data loading failed. table not found: $dbName.$tableName")
- }
val carbonProperty: CarbonProperties = CarbonProperties.getInstance()
carbonProperty.addProperty("zookeeper.enable.lock", "false")
@@ -105,18 +97,30 @@ case class LoadTableCommand(
// update the property with new value
carbonProperty.addProperty(CarbonCommonConstants.NUM_CORES_LOADING, numCoresLoading)
- val optionsFinal = DataLoadingUtil.getDataLoadingOptions(carbonProperty, options)
-
- val tableProperties = relation.tableMeta.carbonTable.getTableInfo
- .getFactTable.getTableProperties
+ try {
+ val table = if (tableInfoOp.isDefined) {
+ CarbonTable.buildFromTableInfo(tableInfoOp.get)
+ } else {
+ val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
+ if (relation == null) {
+ sys.error(s"Table $dbName.$tableName does not exist")
+ }
+ if (null == relation.carbonTable) {
+ LOGGER.error(s"Data loading failed. table not found: $dbName.$tableName")
+ LOGGER.audit(s"Data loading failed. table not found: $dbName.$tableName")
+ sys.error(s"Data loading failed. table not found: $dbName.$tableName")
+ }
+ relation.carbonTable
+ }
- optionsFinal.put("sort_scope", tableProperties.getOrDefault("sort_scope",
- carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
- carbonProperty.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))))
+ val tableProperties = table.getTableInfo.getFactTable.getTableProperties
+ val optionsFinal = DataLoadingUtil.getDataLoadingOptions(carbonProperty, options)
+ optionsFinal.put("sort_scope", tableProperties.getOrDefault("sort_scope",
+ carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
+ carbonProperty.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))))
- try {
- val table = relation.tableMeta.carbonTable
val carbonLoadModel = new CarbonLoadModel()
val factPath = if (dataFrame.isDefined) {
""
@@ -137,11 +141,9 @@ case class LoadTableCommand(
// First system has to partition the data first and then call the load data
LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
GlobalDictionaryUtil.updateTableMetadataFunc = updateTableMetadata
- val storePath = relation.tableMeta.storePath
// add the start entry for the new load in the table status file
if (updateModel.isEmpty) {
- CommonUtil.
- readAndUpdateLoadProgressInTableMeta(carbonLoadModel, storePath, isOverwriteTable)
+ CommonUtil.readAndUpdateLoadProgressInTableMeta(carbonLoadModel, isOverwriteTable)
}
if (isOverwriteTable) {
LOGGER.info(s"Overwrite of carbon table with $dbName.$tableName is in progress")
@@ -158,8 +160,7 @@ case class LoadTableCommand(
carbonLoadModel.setUseOnePass(false)
}
// Create table and metadata folders if not exist
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(storePath, table.getCarbonTableIdentifier)
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier)
val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
val fileType = FileFactory.getFileType(metadataDirectoryPath)
if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
@@ -192,9 +193,9 @@ case class LoadTableCommand(
} finally {
// Once the data load is successful delete the unwanted partition files
try {
- val partitionLocation = relation.tableMeta.storePath + "/partition/" +
+ val partitionLocation = CarbonProperties.getStorePath + "/partition/" +
table.getDatabaseName + "/" +
- table.getFactTableName + "/"
+ table.getTableName + "/"
val fileType = FileFactory.getFileType(partitionLocation)
if (FileFactory.isFileExist(partitionLocation, fileType)) {
val file = FileFactory
@@ -234,7 +235,7 @@ case class LoadTableCommand(
.getCarbonTablePath(carbonLoadModel.getTablePath, carbonTableIdentifier)
val dictFolderPath = carbonTablePath.getMetadataDirectoryPath
val dimensions = carbonTable.getDimensionByTableName(
- carbonTable.getFactTableName).asScala.toArray
+ carbonTable.getTableName).asScala.toArray
val colDictFilePath = carbonLoadModel.getColDictFilePath
if (!StringUtils.isEmpty(colDictFilePath)) {
carbonLoadModel.initPredefDictMap()
@@ -378,8 +379,7 @@ case class LoadTableCommand(
val identifier = model.table.getCarbonTableIdentifier
// update CarbonDataLoadSchema
val carbonTable = metastore.lookupRelation(Option(identifier.getDatabaseName),
- identifier.getTableName)(sqlContext.sparkSession).asInstanceOf[CarbonRelation].tableMeta
- .carbonTable
+ identifier.getTableName)(sqlContext.sparkSession).asInstanceOf[CarbonRelation].carbonTable
carbonLoadModel.setCarbonDataLoadSchema(new CarbonDataLoadSchema(carbonTable))
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
index a52008a..efb6796 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
@@ -31,7 +31,6 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{CarbonEnv, GetDB, Row, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.ExecutionErrors
-import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -64,24 +63,18 @@ object DeleteExecution {
sparkSession: SparkSession,
dataRdd: RDD[Row],
timestamp: String,
- relation: CarbonRelation,
isUpdateOperation: Boolean,
- executorErrors: ExecutionErrors
- ): Boolean = {
+ executorErrors: ExecutionErrors): Boolean = {
var res: Array[List[(SegmentStatus, (SegmentUpdateDetails, ExecutionErrors))]] = null
val tableName = getTableIdentifier(identifier).table
val database = GetDB.getDatabaseName(getTableIdentifier(identifier).database, sparkSession)
- val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(DeleteExecution.getTableIdentifier(identifier))(sparkSession).
- asInstanceOf[CarbonRelation]
-
- val absoluteTableIdentifier = relation.tableMeta.carbonTable.getAbsoluteTableIdentifier
+ val carbonTable = CarbonEnv.getCarbonTable(Some(database), tableName)(sparkSession)
+ val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
val carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier)
val factPath = carbonTablePath.getFactDir
- val carbonTable = relation.tableMeta.carbonTable
var deleteStatus = true
val deleteRdd = if (isUpdateOperation) {
val schema =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala
index 34daf4e..6762489 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala
@@ -54,7 +54,7 @@ object HorizontalCompaction {
}
var compactionTypeIUD = CompactionType.IUD_UPDDEL_DELTA_COMPACTION
- val carbonTable = carbonRelation.tableMeta.carbonTable
+ val carbonTable = carbonRelation.carbonTable
val absTableIdentifier = carbonTable.getAbsoluteTableIdentifier
val updateTimeStamp = System.currentTimeMillis()
// To make sure that update and delete timestamps are not same,
@@ -116,7 +116,7 @@ object HorizontalCompaction {
factTimeStamp: Long,
segLists: util.List[String]): Unit = {
val db = carbonTable.getDatabaseName
- val table = carbonTable.getFactTableName
+ val table = carbonTable.getTableName
// get the valid segments qualified for update compaction.
val validSegList = CarbonDataMergerUtil.getSegListIUDCompactionQualified(segLists,
absTableIdentifier,
@@ -133,7 +133,7 @@ object HorizontalCompaction {
try {
// Update Compaction.
val alterTableModel = AlterTableModel(Option(carbonTable.getDatabaseName),
- carbonTable.getFactTableName,
+ carbonTable.getTableName,
Some(segmentUpdateStatusManager),
CompactionType.IUD_UPDDEL_DELTA_COMPACTION.toString,
Some(factTimeStamp),
@@ -167,7 +167,7 @@ object HorizontalCompaction {
segLists: util.List[String]): Unit = {
val db = carbonTable.getDatabaseName
- val table = carbonTable.getFactTableName
+ val table = carbonTable.getTableName
val deletedBlocksList = CarbonDataMergerUtil.getSegListIUDCompactionQualified(segLists,
absTableIdentifier,
segmentUpdateStatusManager,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala
index b18ab78..5817d88 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala
@@ -60,7 +60,7 @@ object IUDCommonUtil {
logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonTable
.getDatabaseName + "." +
logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonTable
- .getFactTableName
+ .getTableName
val sementProperty = carbonProperties
.getProperty(CarbonCommonConstants.CARBON_INPUT_SEGMENTS + dbAndTb, "")
if (!(sementProperty.equals("") || sementProperty.trim.equals("*"))) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForDeleteCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForDeleteCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForDeleteCommand.scala
index a898822..cf5bfd8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForDeleteCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForDeleteCommand.scala
@@ -51,7 +51,7 @@ private[sql] case class ProjectForDeleteCommand(
val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
.lookupRelation(DeleteExecution.getTableIdentifier(identifier))(sparkSession).
asInstanceOf[CarbonRelation]
- val carbonTable = relation.tableMeta.carbonTable
+ val carbonTable = relation.carbonTable
// trigger event for Delete from table
val operationContext = new OperationContext
@@ -77,9 +77,8 @@ private[sql] case class ProjectForDeleteCommand(
// handle the clean up of IUD.
CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, false)
- if (DeleteExecution
- .deleteDeltaExecution(identifier, sparkSession, dataRdd, timestamp, relation,
- isUpdateOperation = false, executorErrors)) {
+ if (DeleteExecution.deleteDeltaExecution(identifier, sparkSession, dataRdd, timestamp,
+ isUpdateOperation = false, executorErrors)) {
// call IUD Compaction.
HorizontalCompaction.tryHorizontalCompaction(sparkSession, relation,
isUpdateOperation = false)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForUpdateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForUpdateCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForUpdateCommand.scala
index 549c58f..da62f27 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForUpdateCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/ProjectForUpdateCommand.scala
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus, UpdateTablePostEvent, UpdateTablePreEvent}
import org.apache.carbondata.processing.loading.FailureCauses
@@ -58,7 +57,7 @@ private[sql] case class ProjectForUpdateCommand(
val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
.lookupRelation(DeleteExecution.getTableIdentifier(tableIdentifier))(sparkSession).
asInstanceOf[CarbonRelation]
- val carbonTable = relation.tableMeta.carbonTable
+ val carbonTable = relation.carbonTable
// trigger event for Update table
val operationContext = new OperationContext
@@ -74,7 +73,7 @@ private[sql] case class ProjectForUpdateCommand(
val currentTime = CarbonUpdateUtil.readCurrentTime
// var dataFrame: DataFrame = null
var dataSet: DataFrame = null
- val isPersistEnabled = CarbonProperties.getInstance.isPersistUpdateDataset()
+ val isPersistEnabled = CarbonProperties.getInstance.isPersistUpdateDataset
try {
lockStatus = metadataLock.lockWithRetries()
if (lockStatus) {
@@ -83,7 +82,6 @@ private[sql] case class ProjectForUpdateCommand(
else {
throw new Exception("Table is locked for updation. Please try after some time")
}
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
// Get RDD.
dataSet = if (isPersistEnabled) {
@@ -93,7 +91,7 @@ private[sql] case class ProjectForUpdateCommand(
else {
Dataset.ofRows(sparkSession, plan)
}
- var executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
+ val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
// handle the clean up of IUD.
@@ -101,8 +99,7 @@ private[sql] case class ProjectForUpdateCommand(
// do delete operation.
DeleteExecution.deleteDeltaExecution(tableIdentifier, sparkSession, dataSet.rdd,
- currentTime + "",
- relation, isUpdateOperation = true, executionErrors)
+ currentTime + "", isUpdateOperation = true, executionErrors)
if(executionErrors.failureCauses != FailureCauses.NONE) {
throw new Exception(executionErrors.errorMsg)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableDropCarbonPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableDropCarbonPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableDropCarbonPartitionCommand.scala
index acd9bd3..5a0e4cc 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableDropCarbonPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableDropCarbonPartitionCommand.scala
@@ -65,8 +65,7 @@ case class AlterTableDropCarbonPartitionCommand(
val carbonMetaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val relation = carbonMetaStore.lookupRelation(Option(dbName), tableName)(sparkSession)
.asInstanceOf[CarbonRelation]
- val carbonTableIdentifier = relation.tableMeta.carbonTableIdentifier
- val tablePath = relation.tableMeta.tablePath
+ val tablePath = relation.carbonTable.getTablePath
carbonMetaStore.checkSchemasModifiedTimeAndReloadTables()
if (relation == null) {
sys.error(s"Table $dbName.$tableName does not exist")
@@ -75,7 +74,7 @@ case class AlterTableDropCarbonPartitionCommand(
LOGGER.error(s"Alter table failed. table not found: $dbName.$tableName")
sys.error(s"Alter table failed. table not found: $dbName.$tableName")
}
- val table = relation.tableMeta.carbonTable
+ val table = relation.carbonTable
val partitionInfo = table.getPartitionInfo(tableName)
if (partitionInfo == null) {
sys.error(s"Table $tableName is not a partition table.")
@@ -101,7 +100,7 @@ case class AlterTableDropCarbonPartitionCommand(
sys.error(s"Dropping range interval partition isn't support yet!")
}
partitionInfo.dropPartition(partitionIndex)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(tablePath, carbonTableIdentifier)
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier)
val schemaFilePath = carbonTablePath.getSchemaFilePath
// read TableInfo
val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTablePath)(sparkSession)
@@ -142,17 +141,13 @@ case class AlterTableDropCarbonPartitionCommand(
locks = AlterTableUtil.validateTableAndAcquireLock(dbName, tableName,
locksToBeAcquired)(sparkSession)
val carbonLoadModel = new CarbonLoadModel()
- val carbonMetaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val relation = carbonMetaStore.lookupRelation(Option(dbName), tableName)(sparkSession)
- .asInstanceOf[CarbonRelation]
- val carbonTableIdentifier = relation.tableMeta.carbonTableIdentifier
- val table = relation.tableMeta.carbonTable
+ val table = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val dataLoadSchema = new CarbonDataLoadSchema(table)
// Need to fill dimension relation
carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
- carbonLoadModel.setTableName(carbonTableIdentifier.getTableName)
- carbonLoadModel.setDatabaseName(carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setTablePath(relation.tableMeta.tablePath)
+ carbonLoadModel.setTableName(table.getTableName)
+ carbonLoadModel.setDatabaseName(table.getDatabaseName)
+ carbonLoadModel.setTablePath(table.getTablePath)
val loadStartTime = CarbonUpdateUtil.readCurrentTime
carbonLoadModel.setFactTimeStamp(loadStartTime)
alterTableDropPartition(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala
index 0973226..841da67 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/AlterTableSplitCarbonPartitionCommand.scala
@@ -69,8 +69,7 @@ case class AlterTableSplitCarbonPartitionCommand(
val tableName = splitPartitionModel.tableName
val relation = carbonMetaStore.lookupRelation(Option(dbName), tableName)(sparkSession)
.asInstanceOf[CarbonRelation]
- val carbonTableIdentifier = relation.tableMeta.carbonTableIdentifier
- val tablePath = relation.tableMeta.tablePath
+ val tablePath = relation.carbonTable.getTablePath
if (relation == null) {
sys.error(s"Table $dbName.$tableName does not exist")
}
@@ -79,7 +78,7 @@ case class AlterTableSplitCarbonPartitionCommand(
LOGGER.error(s"Alter table failed. table not found: $dbName.$tableName")
sys.error(s"Alter table failed. table not found: $dbName.$tableName")
}
- val table = relation.tableMeta.carbonTable
+ val table = relation.carbonTable
val partitionInfo = table.getPartitionInfo(tableName)
val partitionIds = partitionInfo.getPartitionIds.asScala.map(_.asInstanceOf[Int]).toList
// keep a copy of partitionIdList before update partitionInfo.
@@ -95,7 +94,7 @@ case class AlterTableSplitCarbonPartitionCommand(
updatePartitionInfo(partitionInfo, partitionIds)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(tablePath, carbonTableIdentifier)
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier)
val schemaFilePath = carbonTablePath.getSchemaFilePath
// read TableInfo
val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTablePath)(sparkSession)
@@ -150,16 +149,12 @@ case class AlterTableSplitCarbonPartitionCommand(
locks = AlterTableUtil.validateTableAndAcquireLock(dbName, tableName,
locksToBeAcquired)(sparkSession)
val carbonLoadModel = new CarbonLoadModel()
- val carbonMetaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val relation = carbonMetaStore.lookupRelation(Option(dbName), tableName)(sparkSession)
- .asInstanceOf[CarbonRelation]
- val tablePath = relation.tableMeta.tablePath
- val table = relation.tableMeta.carbonTable
- val carbonTableIdentifier = relation.tableMeta.carbonTableIdentifier
+ val table = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
+ val tablePath = table.getTablePath
val dataLoadSchema = new CarbonDataLoadSchema(table)
carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
- carbonLoadModel.setTableName(carbonTableIdentifier.getTableName)
- carbonLoadModel.setDatabaseName(carbonTableIdentifier.getDatabaseName)
+ carbonLoadModel.setTableName(table.getTableName)
+ carbonLoadModel.setDatabaseName(table.getDatabaseName)
carbonLoadModel.setTablePath(tablePath)
val loadStartTime = CarbonUpdateUtil.readCurrentTime
carbonLoadModel.setFactTimeStamp(loadStartTime)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/ShowCarbonPartitionsCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/ShowCarbonPartitionsCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/ShowCarbonPartitionsCommand.scala
index 224304a..903e93b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/ShowCarbonPartitionsCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/ShowCarbonPartitionsCommand.scala
@@ -41,10 +41,9 @@ private[sql] case class ShowCarbonPartitionsCommand(
override def processSchema(sparkSession: SparkSession): Seq[Row] = {
val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(tableIdentifier)(sparkSession).
- asInstanceOf[CarbonRelation]
- val carbonTable = relation.tableMeta.carbonTable
- val tableName = carbonTable.getFactTableName
+ .lookupRelation(tableIdentifier)(sparkSession).asInstanceOf[CarbonRelation]
+ val carbonTable = relation.carbonTable
+ val tableName = carbonTable.getTableName
val partitionInfo = carbonTable.getPartitionInfo(
carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getTableName)
if (partitionInfo == null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index dd002f0..3854f76 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -73,7 +73,7 @@ case class CreatePreAggregateTableCommand(
// getting the parent table
val parentTable = PreAggregateUtil.getParentCarbonTable(df.logicalPlan)
// getting the table name
- val parentTableName = parentTable.getFactTableName
+ val parentTableName = parentTable.getTableName
// getting the db name of parent table
val parentDbName = parentTable.getDatabaseName
@@ -85,9 +85,8 @@ case class CreatePreAggregateTableCommand(
tableModel.dataMapRelation = Some(fieldRelationMap)
CarbonCreateTableCommand(tableModel).run(sparkSession)
try {
- val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore.
- lookupRelation( tableIdentifier)(sparkSession).asInstanceOf[CarbonRelation]
- val tableInfo = relation.tableMeta.carbonTable.getTableInfo
+ val table = CarbonEnv.getCarbonTable(tableIdentifier)(sparkSession)
+ val tableInfo = table.getTableInfo
// child schema object which will be updated on parent table about the
val childSchema = tableInfo.getFactTable
.buildChildSchema(dataMapName, CarbonCommonConstants.AGGREGATIONDATAMAPSCHEMA,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
index 506a405..f64deec 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
@@ -84,7 +84,7 @@ object PreAggregateDataTypeChangePreListener extends OperationEventListener {
if (carbonTable.isChildDataMap) {
throw new UnsupportedOperationException(
s"Cannot change data type for columns in pre-aggregate table ${ carbonTable.getDatabaseName
- }.${ carbonTable.getFactTableName }")
+ }.${ carbonTable.getTableName }")
}
}
}
@@ -102,7 +102,7 @@ object PreAggregateAddColumnsPreListener extends OperationEventListener {
if (carbonTable.isChildDataMap) {
throw new UnsupportedOperationException(
s"Cannot add columns in pre-aggreagate table ${ carbonTable.getDatabaseName
- }.${ carbonTable.getFactTableName }")
+ }.${ carbonTable.getTableName }")
}
}
}
@@ -185,7 +185,7 @@ object PreAggregateDropColumnPreListener extends OperationEventListener {
}
if (carbonTable.isChildDataMap) {
throw new UnsupportedOperationException(s"Cannot drop columns in pre-aggreagate table ${
- carbonTable.getDatabaseName}.${ carbonTable.getFactTableName }")
+ carbonTable.getDatabaseName}.${ carbonTable.getTableName }")
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index 3193310..1647f9e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -326,21 +326,19 @@ object PreAggregateUtil {
var numberOfCurrentChild: Int = 0
try {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- carbonTable = metastore
- .lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
- .tableMeta.carbonTable
+ carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
locks = acquireLock(dbName, tableName, locksToBeAcquired, carbonTable)
// get the latest carbon table and check for column existence
// read the latest schema file
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(
+ carbonTable.getAbsoluteTableIdentifier)
val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
- val wrapperTableInfo = schemaConverter
- .fromExternalToWrapperTableInfo(thriftTableInfo,
- dbName,
- tableName,
- carbonTable.getTablePath)
+ val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
+ thriftTableInfo,
+ dbName,
+ tableName,
+ carbonTable.getTablePath)
numberOfCurrentChild = wrapperTableInfo.getDataMapSchemaList.size
if (wrapperTableInfo.getDataMapSchemaList.asScala.
exists(f => f.getDataMapName.equalsIgnoreCase(childSchema.getDataMapName))) {
@@ -374,7 +372,7 @@ object PreAggregateUtil {
def updateSchemaInfo(carbonTable: CarbonTable,
thriftTable: TableInfo)(sparkSession: SparkSession): Unit = {
val dbName = carbonTable.getDatabaseName
- val tableName = carbonTable.getFactTableName
+ val tableName = carbonTable.getTableName
CarbonEnv.getInstance(sparkSession).carbonMetastore
.updateTableSchemaForDataMap(carbonTable.getCarbonTableIdentifier,
carbonTable.getCarbonTableIdentifier,
@@ -435,31 +433,30 @@ object PreAggregateUtil {
def revertMainTableChanges(dbName: String, tableName: String, numberOfChildSchema: Int)
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val carbonTable = metastore
- .lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation].tableMeta
- .carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
carbonTable.getTableLastUpdatedTime
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
if (thriftTable.dataMapSchemas.size > numberOfChildSchema) {
- metastore
- .revertTableSchemaForPreAggCreationFailure(carbonTable.getAbsoluteTableIdentifier,
- thriftTable)(sparkSession)
+ metastore.revertTableSchemaForPreAggCreationFailure(
+ carbonTable.getAbsoluteTableIdentifier, thriftTable)(sparkSession)
}
}
def getChildCarbonTable(databaseName: String, tableName: String)
(sparkSession: SparkSession): Option[CarbonTable] = {
val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- metaStore.getTableFromMetadataCache(databaseName, tableName) match {
- case Some(tableMeta) => Some(tableMeta.carbonTable)
- case None => try {
+ val carbonTable = metaStore.getTableFromMetadataCache(databaseName, tableName)
+ if (carbonTable.isEmpty) {
+ try {
Some(metaStore.lookupRelation(Some(databaseName), tableName)(sparkSession)
.asInstanceOf[CarbonRelation].metaData.carbonTable)
} catch {
case _: Exception =>
None
}
+ } else {
+ carbonTable
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
index 2132131..3b39334 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
@@ -57,9 +57,7 @@ private[sql] case class CarbonAlterTableAddColumnCommand(
// older carbon table and this can lead to inconsistent state in the system. Therefor look
// up relation should be called after acquiring the lock
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- carbonTable = metastore
- .lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
- .tableMeta.carbonTable
+ carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val alterTableAddColumnListener = AlterTableAddColumnPreEvent(carbonTable,
alterTableAddColumnsModel)
OperationListenerBus.getInstance().fireEvent(alterTableAddColumnListener)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
index e44899e..c24a8e9 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
@@ -51,9 +51,7 @@ private[sql] case class CarbonAlterTableDataTypeChangeCommand(
locks = AlterTableUtil
.validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- carbonTable = metastore
- .lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
- .tableMeta.carbonTable
+ carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val alterTableDataTypeChangeListener = AlterTableDataTypeChangePreEvent(carbonTable,
alterTableDataTypeChangeModel)
OperationListenerBus.getInstance().fireEvent(alterTableDataTypeChangeListener)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
index dae2d7b..721dd0a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
@@ -53,9 +53,7 @@ private[sql] case class CarbonAlterTableDropColumnCommand(
locks = AlterTableUtil
.validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- carbonTable = metastore
- .lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
- .tableMeta.carbonTable
+ carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val partitionInfo = carbonTable.getPartitionInfo(tableName)
if (partitionInfo != null) {
val partitionColumnSchemaList = partitionInfo.getColumnSchemaList.asScala
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index f1cce13..e7beedd 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -81,9 +81,8 @@ private[sql] case class CarbonAlterTableRenameCommand(
locks = AlterTableUtil
.validateTableAndAcquireLock(oldDatabaseName, oldTableName, locksToBeAcquired)(
sparkSession)
- val tableMeta = metastore.lookupRelation(Some(oldDatabaseName), oldTableName)(sparkSession)
- .asInstanceOf[CarbonRelation].tableMeta
- carbonTable = tableMeta.carbonTable
+ carbonTable = metastore.lookupRelation(Some(oldDatabaseName), oldTableName)(sparkSession)
+ .asInstanceOf[CarbonRelation].carbonTable
// invalid data map for the old table, see CARBON-1690
val oldTableIdentifier = carbonTable.getAbsoluteTableIdentifier
DataMapStoreManager.getInstance().clearDataMaps(oldTableIdentifier)
@@ -134,7 +133,7 @@ private[sql] case class CarbonAlterTableRenameCommand(
carbonTable.getCarbonTableIdentifier,
tableInfo,
schemaEvolutionEntry,
- tableMeta.tablePath)(sparkSession)
+ carbonTable.getTablePath)(sparkSession)
val alterTableRenamePostEvent: AlterTableRenamePostEvent = AlterTableRenamePostEvent(
carbonTable,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index c126b25..a060833 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -357,11 +357,11 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
private def getPartitioning(carbonTable: CarbonTable,
output: Seq[Attribute]): Partitioning = {
- val info: BucketingInfo = carbonTable.getBucketingInfo(carbonTable.getFactTableName)
+ val info: BucketingInfo = carbonTable.getBucketingInfo(carbonTable.getTableName)
if (info != null) {
val cols = info.getListOfColumns.asScala
val sortColumn = carbonTable.
- getDimensionByTableName(carbonTable.getFactTableName).get(0).getColName
+ getDimensionByTableName(carbonTable.getTableName).get(0).getColName
val numBuckets = info.getNumberOfBuckets
val bucketColumns = cols.flatMap { n =>
val attrRef = output.find(_.name.equalsIgnoreCase(n.getColumnName))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index ef2e0a5..d6450c1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -51,8 +51,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
val dbOption = oldTableIdentifier.database.map(_.toLowerCase)
val tableIdentifier = TableIdentifier(oldTableIdentifier.table.toLowerCase(), dbOption)
val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .tableExists(tableIdentifier)(
- sparkSession)
+ .tableExists(tableIdentifier)(sparkSession)
if (isCarbonTable) {
val renameModel = AlterTableRenameModel(tableIdentifier, newTableIdentifier)
ExecutedCommandExec(CarbonAlterTableRenameCommand(renameModel)) :: Nil
@@ -155,13 +154,13 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
CreateDataSourceTableCommand(updatedCatalog, ignoreIfExists = mode == SaveMode.Ignore)
ExecutedCommandExec(cmd) :: Nil
case AlterTableSetPropertiesCommand(tableName, properties, isView)
- if (CarbonEnv.getInstance(sparkSession).carbonMetastore
- .tableExists(tableName)(sparkSession)) => {
+ if CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .tableExists(tableName)(sparkSession) => {
ExecutedCommandExec(AlterTableSetCommand(tableName, properties, isView)) :: Nil
}
case AlterTableUnsetPropertiesCommand(tableName, propKeys, ifExists, isView)
- if (CarbonEnv.getInstance(sparkSession).carbonMetastore
- .tableExists(tableName)(sparkSession)) => {
+ if CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .tableExists(tableName)(sparkSession) => {
ExecutedCommandExec(AlterTableUnsetCommand(tableName, propKeys, ifExists, isView)) :: Nil
}
case _ => Nil
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
index 9ebf47e..49a57e6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
@@ -21,7 +21,7 @@ import org.apache.spark.sql.{CarbonEnv, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy}
-import org.apache.spark.sql.execution.command.{AlterTableRenameCommand, ExecutedCommandExec}
+import org.apache.spark.sql.execution.command.AlterTableRenameCommand
import org.apache.spark.sql.execution.command.mutation.{DeleteExecution, ProjectForDeleteCommand, ProjectForUpdateCommand}
import org.apache.spark.sql.execution.command.schema.{CarbonAlterTableAddColumnCommand, CarbonAlterTableDataTypeChangeCommand, CarbonAlterTableDropColumnCommand}
import org.apache.spark.sql.hive.CarbonRelation
@@ -76,7 +76,6 @@ private[sql] class StreamingTableStrategy(sparkSession: SparkSession) extends Sp
val streaming = CarbonEnv.getInstance(sparkSession).carbonMetastore
.lookupRelation(tableIdentifier)(sparkSession)
.asInstanceOf[CarbonRelation]
- .tableMeta
.carbonTable
.isStreamingTable
if (streaming) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index 6d80a26..87c919d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.hive
import java.util.UUID
import java.util.concurrent.atomic.AtomicLong
-import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, SparkSession}
@@ -44,13 +43,12 @@ import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.writer.ThriftWriter
import org.apache.carbondata.events.{LookupRelationPostEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
-import org.apache.carbondata.processing.merger.TableMeta
import org.apache.carbondata.spark.util.CarbonSparkUtil
-case class MetaData(var tablesMeta: ArrayBuffer[TableMeta]) {
+case class MetaData(var carbonTables: ArrayBuffer[CarbonTable]) {
// clear the metadata
def clear(): Unit = {
- tablesMeta.clear()
+ carbonTables.clear()
}
}
@@ -80,7 +78,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
System.nanoTime() + ""
}
- val metadata = MetaData(new ArrayBuffer[TableMeta]())
+ val metadata = MetaData(new ArrayBuffer[CarbonTable]())
/**
@@ -98,13 +96,12 @@ class CarbonFileMetastore extends CarbonMetaStore {
val tables = getTableFromMetadataCache(database, tableName)
tables match {
case Some(t) =>
- CarbonRelation(database, tableName,
- CarbonSparkUtil.createSparkMeta(t.carbonTable), t)
+ CarbonRelation(database, tableName, CarbonSparkUtil.createSparkMeta(t), t)
case None =>
readCarbonSchema(absIdentifier) match {
case Some(meta) =>
CarbonRelation(database, tableName,
- CarbonSparkUtil.createSparkMeta(meta.carbonTable), meta)
+ CarbonSparkUtil.createSparkMeta(meta), meta)
case None =>
throw new NoSuchTableException(database, tableName)
}
@@ -151,7 +148,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
val operationContext = new OperationContext
val lookupRelationPostEvent: LookupRelationPostEvent =
LookupRelationPostEvent(
- relation.tableMeta.carbonTable,
+ relation.carbonTable,
sparkSession)
OperationListenerBus.getInstance.fireEvent(lookupRelationPostEvent, operationContext)
relation
@@ -164,10 +161,10 @@ class CarbonFileMetastore extends CarbonMetaStore {
* @param tableName
* @return
*/
- def getTableFromMetadataCache(database: String, tableName: String): Option[TableMeta] = {
- metadata.tablesMeta
- .find(c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
- c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableName))
+ def getTableFromMetadataCache(database: String, tableName: String): Option[CarbonTable] = {
+ metadata.carbonTables
+ .find(table => table.getDatabaseName.equalsIgnoreCase(database) &&
+ table.getTableName.equalsIgnoreCase(tableName))
}
def tableExists(
@@ -187,7 +184,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
true
}
- private def readCarbonSchema(identifier: AbsoluteTableIdentifier): Option[TableMeta] = {
+ private def readCarbonSchema(identifier: AbsoluteTableIdentifier): Option[CarbonTable] = {
val dbName = identifier.getCarbonTableIdentifier.getDatabaseName
val tableName = identifier.getCarbonTableIdentifier.getTableName
val tablePath = identifier.getTablePath
@@ -210,12 +207,8 @@ class CarbonFileMetastore extends CarbonMetaStore {
.setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- val tableMeta = new TableMeta(carbonTable.getCarbonTableIdentifier,
- identifier.getTablePath,
- identifier.getTablePath,
- carbonTable)
- metadata.tablesMeta += tableMeta
- Some(tableMeta)
+ metadata.carbonTables += carbonTable
+ Some(carbonTable)
} else {
None
}
@@ -378,10 +371,8 @@ class CarbonFileMetastore extends CarbonMetaStore {
CarbonMetadata.getInstance.removeTable(tableInfo.getTableUniqueName)
removeTableFromMetadata(identifier.getDatabaseName, identifier.getTableName)
CarbonMetadata.getInstance().loadTableMetadata(tableInfo)
- val tableMeta = new TableMeta(identifier, absoluteTableIdentifier.getTablePath,
- absoluteTableIdentifier.getTablePath,
- CarbonMetadata.getInstance().getCarbonTable(identifier.getTableUniqueName))
- metadata.tablesMeta += tableMeta
+ metadata.carbonTables +=
+ CarbonMetadata.getInstance().getCarbonTable(identifier.getTableUniqueName)
}
/**
@@ -391,10 +382,10 @@ class CarbonFileMetastore extends CarbonMetaStore {
* @param tableName
*/
def removeTableFromMetadata(dbName: String, tableName: String): Unit = {
- val metadataToBeRemoved: Option[TableMeta] = getTableFromMetadataCache(dbName, tableName)
- metadataToBeRemoved match {
- case Some(tableMeta) =>
- metadata.tablesMeta -= tableMeta
+ val carbonTableToBeRemoved: Option[CarbonTable] = getTableFromMetadataCache(dbName, tableName)
+ carbonTableToBeRemoved match {
+ case Some(carbonTable) =>
+ metadata.carbonTables -= carbonTable
CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
case None =>
if (LOGGER.isDebugEnabled) {
@@ -409,10 +400,10 @@ class CarbonFileMetastore extends CarbonMetaStore {
CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(
wrapperTableInfo.getTableUniqueName)
- for (i <- metadata.tablesMeta.indices) {
+ for (i <- metadata.carbonTables.indices) {
if (wrapperTableInfo.getTableUniqueName.equals(
- metadata.tablesMeta(i).carbonTableIdentifier.getTableUniqueName)) {
- metadata.tablesMeta(i).carbonTable = carbonTable
+ metadata.carbonTables(i).getTableUniqueName)) {
+ metadata.carbonTables(i) = carbonTable
}
}
}
@@ -434,8 +425,8 @@ class CarbonFileMetastore extends CarbonMetaStore {
def isTablePathExists(tableIdentifier: TableIdentifier)(sparkSession: SparkSession): Boolean = {
try {
- val tablePath = lookupRelation(tableIdentifier)(sparkSession).
- asInstanceOf[CarbonRelation].tableMeta.tablePath
+ val tablePath = lookupRelation(tableIdentifier)(sparkSession)
+ .asInstanceOf[CarbonRelation].carbonTable.getTablePath
val fileType = FileFactory.getFileType(tablePath)
FileFactory.isFileExist(tablePath, fileType)
} catch {
@@ -531,13 +522,13 @@ class CarbonFileMetastore extends CarbonMetaStore {
}
private def refreshCache() {
- metadata.tablesMeta.clear()
+ metadata.carbonTables.clear()
}
override def isReadFromHiveMetaStore: Boolean = false
override def listAllTables(sparkSession: SparkSession): Seq[CarbonTable] =
- metadata.tablesMeta.map(_.carbonTable)
+ metadata.carbonTables
override def getThriftTableInfo(tablePath: CarbonTablePath)
(sparkSession: SparkSession): TableInfo = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
index dedaf1c..4d4229a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
@@ -21,20 +21,17 @@ import scala.collection.JavaConverters._
import org.apache.hadoop.fs.Path
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
import org.apache.carbondata.core.cache.dictionary.ManageDictionaryAndBTree
import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
-import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, RelationIdentifier}
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.format
import org.apache.carbondata.format.SchemaEvolutionEntry
-import org.apache.carbondata.processing.merger.TableMeta
-import org.apache.carbondata.spark.util.{CarbonSparkUtil, CommonUtil}
+import org.apache.carbondata.spark.util.CarbonSparkUtil
/**
* Metastore to store carbonschema in hive
@@ -56,10 +53,8 @@ class CarbonHiveMetaStore extends CarbonFileMetastore {
val info = CarbonUtil.convertGsonToTableInfo(parameters.asJava)
if (info != null) {
val table = CarbonTable.buildFromTableInfo(info)
- val meta = new TableMeta(table.getCarbonTableIdentifier,
- absIdentifier.getTablePath, absIdentifier.getTablePath, table)
CarbonRelation(info.getDatabaseName, info.getFactTable.getTableName,
- CarbonSparkUtil.createSparkMeta(table), meta)
+ CarbonSparkUtil.createSparkMeta(table), table)
} else {
super.createCarbonRelation(parameters, absIdentifier, sparkSession)
}
@@ -107,7 +102,7 @@ class CarbonHiveMetaStore extends CarbonFileMetastore {
val schemaConverter = new ThriftWrapperSchemaConverterImpl
schemaConverter.fromWrapperToExternalTableInfo(carbonTable.getTableInfo,
carbonTable.getDatabaseName,
- carbonTable.getFactTableName)
+ carbonTable.getTableName)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
index 357a812..696342f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
-import org.apache.carbondata.processing.merger.TableMeta
+
/**
* Interface for Carbonmetastore
@@ -140,7 +140,7 @@ trait CarbonMetaStore {
def getThriftTableInfo(tablePath: CarbonTablePath)(sparkSession: SparkSession): TableInfo
- def getTableFromMetadataCache(database: String, tableName: String): Option[TableMeta]
+ def getTableFromMetadataCache(database: String, tableName: String): Option[CarbonTable]
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index 3fb0db0..c48e6e8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -761,7 +761,7 @@ object CarbonPreInsertionCasts extends Rule[LogicalPlan] {
.DEFAULT_MAX_NUMBER_OF_COLUMNS
)
}
- val isAggregateTable = !relation.carbonRelation.tableMeta.carbonTable.getTableInfo
+ val isAggregateTable = !relation.carbonRelation.carbonTable.getTableInfo
.getParentRelationIdentifiers.isEmpty
// transform logical plan if the load is for aggregate table.
val childPlan = if (isAggregateTable) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
index 2c476ed..9187fe2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
@@ -29,10 +29,10 @@ import org.apache.spark.sql.types._
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.processing.merger.TableMeta
/**
* Represents logical plan for one carbon table
@@ -41,7 +41,7 @@ case class CarbonRelation(
databaseName: String,
tableName: String,
var metaData: CarbonMetaData,
- tableMeta: TableMeta)
+ carbonTable: CarbonTable)
extends LeafNode with MultiInstanceRelation {
def recursiveMethod(dimName: String, childDim: CarbonDimension): String = {
@@ -84,17 +84,17 @@ case class CarbonRelation(
}
override def newInstance(): LogicalPlan = {
- CarbonRelation(databaseName, tableName, metaData, tableMeta)
+ CarbonRelation(databaseName, tableName, metaData, carbonTable)
.asInstanceOf[this.type]
}
- val dimensionsAttr = {
+ val dimensionsAttr: Seq[AttributeReference] = {
val sett = new LinkedHashSet(
- tableMeta.carbonTable.getDimensionByTableName(tableMeta.carbonTableIdentifier.getTableName)
+ carbonTable.getDimensionByTableName(carbonTable.getTableName)
.asScala.asJava)
sett.asScala.toSeq.map(dim => {
val dimval = metaData.carbonTable
- .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName)
+ .getDimensionByName(metaData.carbonTable.getTableName, dim.getColName)
val output: DataType = dimval.getDataType.getName.toLowerCase match {
case "array" =>
CarbonMetastoreTypes.toDataType(s"array<${ getArrayChildren(dim.getColName) }>")
@@ -113,11 +113,10 @@ case class CarbonRelation(
}
val measureAttr = {
- val factTable = tableMeta.carbonTable.getFactTableName
+ val factTable = carbonTable.getTableName
new LinkedHashSet(
- tableMeta.carbonTable.
- getMeasureByTableName(tableMeta.carbonTable.getFactTableName).
- asScala.asJava).asScala.toSeq.map { x =>
+ carbonTable.getMeasureByTableName(carbonTable.getTableName).asScala.asJava).asScala.toSeq
+ .map { x =>
val metastoreType = metaData.carbonTable.getMeasureByName(factTable, x.getColName)
.getDataType.getName.toLowerCase match {
case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"
@@ -131,7 +130,7 @@ case class CarbonRelation(
}
override val output = {
- val columns = tableMeta.carbonTable.getCreateOrderColumn(tableMeta.carbonTable.getFactTableName)
+ val columns = carbonTable.getCreateOrderColumn(carbonTable.getTableName)
.asScala
// convert each column to Attribute
columns.filter(!_.isInvisible).map { column =>
@@ -196,12 +195,11 @@ case class CarbonRelation(
def sizeInBytes: Long = {
val tableStatusNewLastUpdatedTime = SegmentStatusManager.getTableStatusLastModifiedTime(
- tableMeta.carbonTable.getAbsoluteTableIdentifier)
+ carbonTable.getAbsoluteTableIdentifier)
if (tableStatusLastUpdateTime != tableStatusNewLastUpdatedTime) {
val tablePath = CarbonStorePath.getCarbonTablePath(
- tableMeta.storePath,
- tableMeta.carbonTableIdentifier).getPath
+ carbonTable.getAbsoluteTableIdentifier).getPath
val fileType = FileFactory.getFileType(tablePath)
if(FileFactory.isFileExist(tablePath, fileType)) {
tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonSessionState.scala
index e587395..b0aecd7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.parser.CarbonSparkSqlParser
import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events._
/**
@@ -106,15 +107,15 @@ class CarbonSessionCatalog(
alias: Option[String],
carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation): Boolean = {
var isRefreshed = false
- val storePath = CarbonEnv.getInstance(sparkSession).storePath
+ val storePath = CarbonProperties.getStorePath
carbonEnv.carbonMetastore.
checkSchemasModifiedTimeAndReloadTables()
- val tableMeta = carbonEnv.carbonMetastore
- .getTableFromMetadataCache(carbonDatasourceHadoopRelation.carbonTable.getDatabaseName,
- carbonDatasourceHadoopRelation.carbonTable.getFactTableName)
- if (tableMeta.isEmpty || (tableMeta.isDefined &&
- tableMeta.get.carbonTable.getTableLastUpdatedTime !=
+ val table = carbonEnv.carbonMetastore.getTableFromMetadataCache(
+ carbonDatasourceHadoopRelation.carbonTable.getDatabaseName,
+ carbonDatasourceHadoopRelation.carbonTable.getTableName)
+ if (table.isEmpty || (table.isDefined &&
+ table.get.getTableLastUpdatedTime !=
carbonDatasourceHadoopRelation.carbonTable.getTableLastUpdatedTime)) {
refreshTable(identifier)
DataMapStoreManager.getInstance().
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
index b76b24e..9cc5d86 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
@@ -42,11 +42,11 @@ case class CarbonDropDatabaseCommand(command: DropDatabaseCommand)
var databaseLocation = ""
try {
databaseLocation = GetDB.getDatabaseLocation(dbName, sparkSession,
- CarbonEnv.getInstance(sparkSession).storePath)
+ CarbonProperties.getStorePath)
} catch {
case e: NoSuchDatabaseException =>
// ignore the exception as exception will be handled by hive command.run
- databaseLocation = CarbonEnv.getInstance(sparkSession).storePath
+ databaseLocation = CarbonProperties.getStorePath
}
// DropHiveDB command will fail if cascade is false and one or more table exists in database
if (command.cascade && tablesInDB != null) {
[26/28] carbondata git commit: [CARBONDATA-1480]Min Max Index Example
for DataMap
Posted by ja...@apache.org.
[CARBONDATA-1480]Min Max Index Example for DataMap
Datamap Example. Implementation of Min Max Index through Datamap. And Using the Index while prunning.
This closes #1359
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/29431084
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/29431084
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/29431084
Branch: refs/heads/fgdatamap
Commit: 2943108433c37b3187d95078f417a59bc66ac841
Parents: 85dc4ff
Author: sounakr <so...@gmail.com>
Authored: Thu Sep 28 16:21:05 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Nov 18 23:22:26 2017 +0800
----------------------------------------------------------------------
.../core/datamap/DataMapStoreManager.java | 16 +-
.../carbondata/core/datamap/TableDataMap.java | 15 +-
.../carbondata/core/datamap/dev/DataMap.java | 3 +-
.../core/datamap/dev/DataMapWriter.java | 3 +-
.../indexstore/SegmentPropertiesFetcher.java | 36 +++
.../blockletindex/BlockletDataMap.java | 2 +-
.../blockletindex/BlockletDataMapFactory.java | 33 ++-
datamap/examples/pom.xml | 111 ++++++++++
.../datamap/examples/BlockletMinMax.java | 41 ++++
.../datamap/examples/MinMaxDataMap.java | 143 ++++++++++++
.../datamap/examples/MinMaxDataMapFactory.java | 114 ++++++++++
.../datamap/examples/MinMaxDataWriter.java | 221 +++++++++++++++++++
.../examples/MinMaxIndexBlockDetails.java | 77 +++++++
.../MinMaxDataMapExample.scala | 77 +++++++
.../testsuite/datamap/DataMapWriterSuite.scala | 2 +-
pom.xml | 2 +
.../datamap/DataMapWriterListener.java | 4 +-
.../store/writer/AbstractFactDataWriter.java | 7 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 3 +
19 files changed, 892 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index d30483a..90e5fff 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -103,7 +104,7 @@ public final class DataMapStoreManager {
tableDataMaps = new ArrayList<>();
}
TableDataMap dataMap = getTableDataMap(dataMapName, tableDataMaps);
- if (dataMap != null) {
+ if (dataMap != null && dataMap.getDataMapName().equalsIgnoreCase(dataMapName)) {
throw new RuntimeException("Already datamap exists in that path with type " + dataMapName);
}
@@ -113,12 +114,15 @@ public final class DataMapStoreManager {
DataMapFactory dataMapFactory = factoryClass.newInstance();
dataMapFactory.init(identifier, dataMapName);
BlockletDetailsFetcher blockletDetailsFetcher;
+ SegmentPropertiesFetcher segmentPropertiesFetcher = null;
if (dataMapFactory instanceof BlockletDetailsFetcher) {
blockletDetailsFetcher = (BlockletDetailsFetcher) dataMapFactory;
} else {
blockletDetailsFetcher = getBlockletDetailsFetcher(identifier);
}
- dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory, blockletDetailsFetcher);
+ segmentPropertiesFetcher = (SegmentPropertiesFetcher) blockletDetailsFetcher;
+ dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory, blockletDetailsFetcher,
+ segmentPropertiesFetcher);
} catch (Exception e) {
LOGGER.error(e);
throw new RuntimeException(e);
@@ -128,11 +132,11 @@ public final class DataMapStoreManager {
return dataMap;
}
- private TableDataMap getTableDataMap(String dataMapName,
- List<TableDataMap> tableDataMaps) {
+ private TableDataMap getTableDataMap(String dataMapName, List<TableDataMap> tableDataMaps) {
TableDataMap dataMap = null;
- for (TableDataMap tableDataMap: tableDataMaps) {
- if (tableDataMap.getDataMapName().equals(dataMapName)) {
+ for (TableDataMap tableDataMap : tableDataMaps) {
+ if (tableDataMap.getDataMapName().equals(dataMapName) || (!tableDataMap.getDataMapName()
+ .equals(""))) {
dataMap = tableDataMap;
break;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 705a9fd..07d8eb7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -22,9 +22,11 @@ import java.util.List;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.events.Event;
@@ -45,15 +47,19 @@ public final class TableDataMap implements OperationEventListener {
private BlockletDetailsFetcher blockletDetailsFetcher;
+ private SegmentPropertiesFetcher segmentPropertiesFetcher;
+
/**
* It is called to initialize and load the required table datamap metadata.
*/
public TableDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
- DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher) {
+ DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher,
+ SegmentPropertiesFetcher segmentPropertiesFetcher) {
this.identifier = identifier;
this.dataMapName = dataMapName;
this.dataMapFactory = dataMapFactory;
this.blockletDetailsFetcher = blockletDetailsFetcher;
+ this.segmentPropertiesFetcher = segmentPropertiesFetcher;
}
/**
@@ -66,11 +72,13 @@ public final class TableDataMap implements OperationEventListener {
public List<ExtendedBlocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp)
throws IOException {
List<ExtendedBlocklet> blocklets = new ArrayList<>();
+ SegmentProperties segmentProperties;
for (String segmentId : segmentIds) {
List<Blocklet> pruneBlocklets = new ArrayList<>();
List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
+ segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId);
for (DataMap dataMap : dataMaps) {
- pruneBlocklets.addAll(dataMap.prune(filterExp));
+ pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties));
}
blocklets.addAll(addSegmentId(blockletDetailsFetcher
.getExtendedBlocklets(pruneBlocklets, segmentId), segmentId));
@@ -122,7 +130,8 @@ public final class TableDataMap implements OperationEventListener {
List<Blocklet> blocklets = new ArrayList<>();
List<DataMap> dataMaps = dataMapFactory.getDataMaps(distributable);
for (DataMap dataMap : dataMaps) {
- blocklets.addAll(dataMap.prune(filterExp));
+ blocklets.addAll(dataMap.prune(filterExp,
+ segmentPropertiesFetcher.getSegmentProperties(distributable.getSegmentId())));
}
for (Blocklet blocklet: blocklets) {
ExtendedBlocklet detailedBlocklet =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index ada23ad..3731922 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datamap.dev;
import java.io.IOException;
import java.util.List;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -40,7 +41,7 @@ public interface DataMap {
* @param filterExp
* @return
*/
- List<Blocklet> prune(FilterResolverIntf filterExp);
+ List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties);
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
index 28163d7..413eaa5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
@@ -27,7 +27,7 @@ public interface DataMapWriter {
* Start of new block notification.
* @param blockId file name of the carbondata file
*/
- void onBlockStart(String blockId);
+ void onBlockStart(String blockId, String blockPath);
/**
* End of block notification
@@ -45,7 +45,6 @@ public interface DataMapWriter {
* @param blockletId sequence number of blocklet in the block
*/
void onBlockletEnd(int blockletId);
-
/**
* Add the column pages row to the datamap, order of pages is same as `indexColumns` in
* DataMapMeta returned in DataMapFactory.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java b/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java
new file mode 100644
index 0000000..ec2ae93
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.indexstore;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+
+/**
+ * Fetches the detailed segmentProperties which has more information to execute the query
+ */
+public interface SegmentPropertiesFetcher {
+
+ /**
+ * get the Segment properties based on the SegmentID.
+ * @param segmentId
+ * @return
+ * @throws IOException
+ */
+ SegmentProperties getSegmentProperties(String segmentId) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 43e265d..54016ad 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -315,7 +315,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
@Override
- public List<Blocklet> prune(FilterResolverIntf filterExp) {
+ public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties) {
// getting the start and end index key based on filter for hitting the
// selected block reference nodes based on filter resolver tree.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 2e2cab5..61e5ceb 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -30,13 +30,18 @@ import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -48,13 +53,17 @@ import org.apache.hadoop.fs.RemoteIterator;
/**
* Table map for blocklet
*/
-public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFetcher {
+public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFetcher,
+ SegmentPropertiesFetcher {
private AbsoluteTableIdentifier identifier;
// segmentId -> list of index file
private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
+ // segmentId -> SegmentProperties.
+ private Map<String, SegmentProperties> segmentPropertiesMap = new HashMap<>();
+
private Cache<TableBlockIndexUniqueIdentifier, DataMap> cache;
@Override
@@ -170,6 +179,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
@Override
public void clear(String segmentId) {
+ segmentPropertiesMap.remove(segmentId);
List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segmentId);
if (blockIndexes != null) {
for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
@@ -221,4 +231,25 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
// TODO: pass SORT_COLUMNS into this class
return null;
}
+
+ @Override public SegmentProperties getSegmentProperties(String segmentId) throws IOException {
+ SegmentProperties segmentProperties = segmentPropertiesMap.get(segmentId);
+ if (segmentProperties == null) {
+ int[] columnCardinality;
+ List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
+ getTableBlockIndexUniqueIdentifiers(segmentId);
+ DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
+ List<DataFileFooter> indexInfo =
+ fileFooterConverter.getIndexInfo(tableBlockIndexUniqueIdentifiers.get(0).getFilePath());
+ for (DataFileFooter fileFooter : indexInfo) {
+ List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
+ if (segmentProperties == null) {
+ columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
+ segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
+ }
+ }
+ segmentPropertiesMap.put(segmentId, segmentProperties);
+ }
+ return segmentProperties;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/datamap/examples/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/examples/pom.xml b/datamap/examples/pom.xml
new file mode 100644
index 0000000..6832e62
--- /dev/null
+++ b/datamap/examples/pom.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-parent</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>carbondata-datamap-examples</artifactId>
+ <name>Apache CarbonData :: Datamap Examples</name>
+
+ <properties>
+ <dev.path>${basedir}/../../dev</dev.path>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-spark2</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive-thriftserver_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-repl_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.10</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-repl_${scala.binary.version}</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/minmaxdatamap/main/java</sourceDirectory>
+ <resources>
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>CARBON_EXAMPLESLogResource.properties</include>
+ </includes>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <version>2.15.2</version>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ <phase>compile</phase>
+ </execution>
+ <execution>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java
new file mode 100644
index 0000000..e6968fe
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+
+public class BlockletMinMax {
+ private byte[][] Min;
+
+ private byte[][] Max;
+
+ public byte[][] getMin() {
+ return Min;
+ }
+
+ public void setMin(byte[][] min) {
+ Min = min;
+ }
+
+ public byte[][] getMax() {
+ return Max;
+ }
+
+ public void setMax(byte[][] max) {
+ Max = max;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
new file mode 100644
index 0000000..2ad6327
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+/**
+ * Datamap implementation for min max blocklet.
+ */
+public class MinMaxDataMap implements DataMap {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(MinMaxDataMap.class.getName());
+
+ public static final String NAME = "clustered.minmax.btree.blocklet";
+
+ private String filePath;
+
+ private MinMaxIndexBlockDetails[] readMinMaxDataMap;
+
+ @Override public void init(String filePath) throws MemoryException, IOException {
+ this.filePath = filePath;
+ CarbonFile[] listFiles = getCarbonMinMaxIndexFiles(filePath, "0");
+ for (int i = 0; i < listFiles.length; i++) {
+ readMinMaxDataMap = readJson(listFiles[i].getPath());
+ }
+ }
+
+ private CarbonFile[] getCarbonMinMaxIndexFiles(String filePath, String segmentId) {
+ String path = filePath.substring(0, filePath.lastIndexOf("/") + 1);
+ CarbonFile carbonFile = FileFactory.getCarbonFile(path);
+ return carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(".minmaxindex");
+ }
+ });
+ }
+
+ public MinMaxIndexBlockDetails[] readJson(String filePath) throws IOException {
+ Gson gsonObjectToRead = new Gson();
+ DataInputStream dataInputStream = null;
+ BufferedReader buffReader = null;
+ InputStreamReader inStream = null;
+ MinMaxIndexBlockDetails[] readMinMax = null;
+ AtomicFileOperations fileOperation =
+ new AtomicFileOperationsImpl(filePath, FileFactory.getFileType(filePath));
+
+ try {
+ if (!FileFactory.isFileExist(filePath, FileFactory.getFileType(filePath))) {
+ return null;
+ }
+ dataInputStream = fileOperation.openForRead();
+ inStream = new InputStreamReader(dataInputStream,
+ CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
+ buffReader = new BufferedReader(inStream);
+ readMinMax = gsonObjectToRead.fromJson(buffReader, MinMaxIndexBlockDetails[].class);
+ } catch (IOException e) {
+ return null;
+ } finally {
+ CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
+ }
+ return readMinMax;
+ }
+
+ /**
+ * Block Prunning logic for Min Max DataMap.
+ *
+ * @param filterExp
+ * @param segmentProperties
+ * @return
+ */
+ @Override public List<Blocklet> prune(FilterResolverIntf filterExp,
+ SegmentProperties segmentProperties) {
+ List<Blocklet> blocklets = new ArrayList<>();
+
+ if (filterExp == null) {
+ for (int i = 0; i < readMinMaxDataMap.length; i++) {
+ blocklets.add(new Blocklet(readMinMaxDataMap[i].getFilePath(),
+ String.valueOf(readMinMaxDataMap[i].getBlockletId())));
+ }
+ } else {
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
+ int startIndex = 0;
+ while (startIndex < readMinMaxDataMap.length) {
+ BitSet bitSet = filterExecuter.isScanRequired(readMinMaxDataMap[startIndex].getMaxValues(),
+ readMinMaxDataMap[startIndex].getMinValues());
+ if (!bitSet.isEmpty()) {
+ blocklets.add(new Blocklet(readMinMaxDataMap[startIndex].getFilePath(),
+ String.valueOf(readMinMaxDataMap[startIndex].getBlockletId())));
+ }
+ startIndex++;
+ }
+ }
+ return blocklets;
+ }
+
+ @Override
+ public void clear() {
+ readMinMaxDataMap = null;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
new file mode 100644
index 0000000..b196d0d
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.events.ChangeEvent;
+import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+
+
+/**
+ * Min Max DataMap Factory
+ */
+public class MinMaxDataMapFactory implements DataMapFactory {
+
+ private AbsoluteTableIdentifier identifier;
+
+ @Override
+ public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ this.identifier = identifier;
+ }
+
+ /**
+ * createWriter will return the MinMaxDataWriter.
+ * @param segmentId
+ * @return
+ */
+ @Override
+ public DataMapWriter createWriter(String segmentId) {
+ return new MinMaxDataWriter();
+ }
+
+ /**
+ * getDataMaps Factory method Initializes the Min Max Data Map and returns.
+ * @param segmentId
+ * @return
+ * @throws IOException
+ */
+ @Override public List<DataMap> getDataMaps(String segmentId) throws IOException {
+ List<DataMap> dataMapList = new ArrayList<>();
+ // Form a dataMap of Type MinMaxDataMap.
+ MinMaxDataMap dataMap = new MinMaxDataMap();
+ try {
+ dataMap.init(identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + File.separator);
+ } catch (MemoryException ex) {
+
+ }
+ dataMapList.add(dataMap);
+ return dataMapList;
+ }
+
+ /**
+ *
+ * @param segmentId
+ * @return
+ */
+ @Override public List<DataMapDistributable> toDistributable(String segmentId) {
+ return null;
+ }
+
+ /**
+ * Clear the DataMap.
+ * @param segmentId
+ */
+ @Override public void clear(String segmentId) {
+ }
+
+ /**
+ * Clearing the data map.
+ */
+ @Override
+ public void clear() {
+ }
+
+ @Override public DataMap getDataMap(DataMapDistributable distributable) {
+ return null;
+ }
+
+ @Override
+ public void fireEvent(ChangeEvent event) {
+
+ }
+
+ @Override
+ public DataMapMeta getMeta() {
+ return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")), FilterType.EQUALTO);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
new file mode 100644
index 0000000..78544d3
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+public class MinMaxDataWriter implements DataMapWriter {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(TableInfo.class.getName());
+
+ private byte[][] pageLevelMin, pageLevelMax;
+
+ private byte[][] blockletLevelMin, blockletLevelMax;
+
+ private Map<Integer, BlockletMinMax> blockMinMaxMap;
+
+ private String blockPath;
+
+
+ @Override public void onBlockStart(String blockId, String blockPath) {
+ pageLevelMax = null;
+ pageLevelMin = null;
+ blockletLevelMax = null;
+ blockletLevelMin = null;
+ blockMinMaxMap = null;
+ blockMinMaxMap = new HashMap<Integer, BlockletMinMax>();
+ this.blockPath = blockPath;
+ }
+
+ @Override public void onBlockEnd(String blockId) {
+ updateMinMaxIndex(blockId);
+ }
+
+ @Override public void onBlockletStart(int blockletId) {
+ }
+
+ @Override public void onBlockletEnd(int blockletId) {
+ updateBlockletMinMax(blockletId);
+ }
+
+ @Override
+ public void onPageAdded(int blockletId, int pageId, ColumnPage[] pages) {
+ // Calculate Min and Max value within this page.
+
+ // As part of example we are extracting Min Max values Manually. The same can be done from
+ // retrieving the page statistics. For e.g.
+
+ // if (pageLevelMin == null && pageLevelMax == null) {
+ // pageLevelMin[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMin());
+ // pageLevelMax[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMax());
+ // } else {
+ // if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMin[1], CarbonUtil
+ // .getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMin())) > 0) {
+ // pageLevelMin[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMin());
+ // }
+ // if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMax[1], CarbonUtil
+ // .getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMax())) < 0) {
+ // pageLevelMax[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMax());
+ // }
+
+ byte[] value = new byte[pages[0].getBytes(0).length - 2];
+ if (pageLevelMin == null && pageLevelMax == null) {
+ pageLevelMin = new byte[2][];
+ pageLevelMax = new byte[2][];
+
+ System.arraycopy(pages[0].getBytes(0), 2, value, 0, value.length);
+ pageLevelMin[1] = value;
+ pageLevelMax[1] = value;
+
+ } else {
+ for (int rowIndex = 0; rowIndex < pages[0].getPageSize(); rowIndex++) {
+ System.arraycopy(pages[0].getBytes(rowIndex), 2, value, 0, value.length);
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMin[1], value) > 0) {
+ pageLevelMin[1] = value;
+ }
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMax[1], value) < 0) {
+ pageLevelMax[1] = value;
+ }
+ }
+ }
+ }
+
+ private void updateBlockletMinMax(int blockletId) {
+ if (blockletLevelMax == null || blockletLevelMin == null) {
+ blockletLevelMax = new byte[2][];
+ blockletLevelMin = new byte[2][];
+ if (pageLevelMax != null || pageLevelMin != null) {
+ blockletLevelMin = pageLevelMin;
+ blockletLevelMax = pageLevelMax;
+ }
+ } else {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockletLevelMin[1], pageLevelMin[1]) > 0) {
+ blockletLevelMin = pageLevelMin;
+ }
+
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockletLevelMax[1], pageLevelMax[1]) > 0) {
+ blockletLevelMax = pageLevelMax;
+ }
+ }
+ BlockletMinMax blockletMinMax = new BlockletMinMax();
+ blockletMinMax.setMax(blockletLevelMax);
+ blockletMinMax.setMin(blockletLevelMin);
+ blockMinMaxMap.put(blockletId, blockletMinMax);
+ }
+
+
+ public void updateMinMaxIndex(String blockId) {
+ constructMinMaxIndex(blockId);
+ }
+
+
+
+ /**
+ * Construct the Min Max Index.
+ * @param blockId
+ */
+ public void constructMinMaxIndex(String blockId) {
+ // construct Min and Max values of each Blocklets present inside a block.
+ List<MinMaxIndexBlockDetails> tempMinMaxIndexBlockDetails = null;
+ tempMinMaxIndexBlockDetails = loadBlockDetails();
+ try {
+ writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockPath, blockId);
+ } catch (IOException ex) {
+ LOGGER.info(" Unable to write the file");
+ }
+ }
+
+ /**
+ * loadBlockDetails into the MinMaxIndexBlockDetails class.
+ */
+ private List<MinMaxIndexBlockDetails> loadBlockDetails() {
+ List<MinMaxIndexBlockDetails> minMaxIndexBlockDetails = new ArrayList<MinMaxIndexBlockDetails>();
+ MinMaxIndexBlockDetails tmpminMaxIndexBlockDetails = new MinMaxIndexBlockDetails();
+
+ for (int index = 0; index < blockMinMaxMap.size(); index++) {
+ tmpminMaxIndexBlockDetails.setMinValues(blockMinMaxMap.get(index).getMin());
+ tmpminMaxIndexBlockDetails.setMaxValues(blockMinMaxMap.get(index).getMax());
+ tmpminMaxIndexBlockDetails.setBlockletId(index);
+ tmpminMaxIndexBlockDetails.setFilePath(this.blockPath);
+ minMaxIndexBlockDetails.add(tmpminMaxIndexBlockDetails);
+ }
+ return minMaxIndexBlockDetails;
+ }
+
+ /**
+ * Write the data to a file. This is JSON format file.
+ * @param minMaxIndexBlockDetails
+ * @param blockPath
+ * @param blockId
+ * @throws IOException
+ */
+ public void writeMinMaxIndexFile(List<MinMaxIndexBlockDetails> minMaxIndexBlockDetails,
+ String blockPath, String blockId) throws IOException {
+ String filePath = blockPath.substring(0, blockPath.lastIndexOf(File.separator) + 1) + blockId
+ + ".minmaxindex";
+ BufferedWriter brWriter = null;
+ DataOutputStream dataOutStream = null;
+ try {
+ FileFactory.createNewFile(filePath, FileFactory.getFileType(filePath));
+ dataOutStream = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath));
+ Gson gsonObjectToWrite = new Gson();
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream,
+ CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT));
+ String minmaxIndexData = gsonObjectToWrite.toJson(minMaxIndexBlockDetails);
+ brWriter.write(minmaxIndexData);
+ } catch (IOException ioe) {
+ LOGGER.info("Error in writing minMaxindex file");
+ } finally {
+ if (null != brWriter) {
+ brWriter.flush();
+ }
+ if (null != dataOutStream) {
+ dataOutStream.flush();
+ }
+ CarbonUtil.closeStreams(brWriter, dataOutStream);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
new file mode 100644
index 0000000..0596db5
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.Serializable;
+
+public class MinMaxIndexBlockDetails implements Serializable {
+ private static final long serialVersionUID = 1206104914911491724L;
+
+ /**
+ * Min value of a column of one blocklet Bit-Packed
+ */
+ private byte[][] minValues;
+
+ /**
+ * Max value of a columns of one blocklet Bit-Packed
+ */
+ private byte[][] maxValues;
+
+ /**
+ * filePath pointing to the block.
+ */
+ private String filePath;
+
+ /**
+ * BlockletID of the block.
+ */
+ private Integer BlockletId;
+
+
+ public byte[][] getMinValues() {
+ return minValues;
+ }
+
+ public void setMinValues(byte[][] minValues) {
+ this.minValues = minValues;
+ }
+
+ public byte[][] getMaxValues() {
+ return maxValues;
+ }
+
+ public void setMaxValues(byte[][] maxValues) {
+ this.maxValues = maxValues;
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
+
+ public Integer getBlockletId() {
+ return BlockletId;
+ }
+
+ public void setBlockletId(Integer blockletId) {
+ BlockletId = blockletId;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala b/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
new file mode 100644
index 0000000..0cfe410
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.datamap.examples
+
+import java.io.File
+
+import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.util.CarbonProperties
+
+object MinMaxDataMapExample {
+ def main(args: Array[String]): Unit = {
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "").getCanonicalPath
+ val storeLocation = s"$rootPath/dataMap/examples/target/store"
+ val warehouse = s"$rootPath/datamap/examples/target/warehouse"
+ val metastoredb = s"$rootPath/datamap/examples/target"
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ import org.apache.spark.sql.CarbonSession._
+
+ val spark = SparkSession
+ .builder()
+ .master("local")
+ .appName("CarbonDataMapExample")
+ .config("spark.sql.warehouse.dir", warehouse)
+ .getOrCreateCarbonSession(storeLocation)
+
+ spark.sparkContext.setLogLevel("ERROR")
+ import spark.implicits._
+
+ // register datamap writer
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ AbsoluteTableIdentifier.from(storeLocation, "default", "carbonminmax"),
+ classOf[MinMaxDataMapFactory].getName,
+ MinMaxDataMap.NAME)
+
+ spark.sql("DROP TABLE IF EXISTS carbonminmax")
+
+ val df = spark.sparkContext.parallelize(1 to 33000)
+ .map(x => ("a", "b", x))
+ .toDF("c1", "c2", "c3")
+
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbonminmax")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ // Query the table.
+ spark.sql("select c2 from carbonminmax").show(20, false)
+ spark.sql("select c2 from carbonminmax where c2 = 'b'").show(20, false)
+ spark.sql("DROP TABLE IF EXISTS carbonminmax")
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index 9545f0f..94fbff4 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -183,7 +183,7 @@ object DataMapWriterSuite {
callbackSeq :+= s"blocklet start $blockletId"
}
- override def onBlockStart(blockId: String): Unit = {
+ override def onBlockStart(blockId: String, blockPath: String): Unit = {
callbackSeq :+= s"block start $blockId"
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 342cb2b..18fd17c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -435,6 +435,7 @@
<module>format</module>
<module>integration/spark2</module>
<module>examples/spark2</module>
+ <module>datamap/examples</module>
<module>integration/hive</module>
<module>integration/presto</module>
<module>examples/flink</module>
@@ -474,6 +475,7 @@
<module>integration/presto</module>
<module>streaming</module>
<module>examples/spark2</module>
+ <module>datamap/examples</module>
</modules>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 4b0113c..8e350d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -82,10 +82,10 @@ public class DataMapWriterListener {
LOG.info("DataMapWriter " + writer + " added");
}
- public void onBlockStart(String blockId) {
+ public void onBlockStart(String blockId, String blockPath) {
for (List<DataMapWriter> writers : registry.values()) {
for (DataMapWriter writer : writers) {
- writer.onBlockStart(blockId);
+ writer.onBlockStart(blockId, blockPath);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 855ec03..b00ec70 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -268,7 +268,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
private void notifyDataMapBlockStart() {
if (listener != null) {
- listener.onBlockStart(carbonDataFileName);
+ listener.onBlockStart(carbonDataFileName, constructFactFileFullPath());
}
}
@@ -279,6 +279,11 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
blockletId = 0;
}
+ private String constructFactFileFullPath() {
+ String factFilePath =
+ this.dataWriterVo.getCarbonDataDirectoryPath() + File.separator + this.carbonDataFileName;
+ return factFilePath;
+ }
/**
* Finish writing current file. It will flush stream, copy and rename temp file to final file
* @param copyInCurrentThread set to false if want to do data copy in a new thread
http://git-wip-us.apache.org/repos/asf/carbondata/blob/29431084/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index d8ae8ff..c366f63 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.processing.store.TablePage;
import org.apache.carbondata.processing.store.writer.AbstractFactDataWriter;
import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
+
/**
* Below class will be used to write the data in V3 format
* <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
@@ -157,6 +158,8 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter {
}
}
+
+
/**
* Write the collect blocklet data (blockletDataHolder) to file
*/
[21/28] carbondata git commit: [CARBONDATA-1651] [Supported Boolean
Type When Saving DataFrame] Provided Support For Boolean Data Type In
CarbonDataFrameWriter
Posted by ja...@apache.org.
[CARBONDATA-1651] [Supported Boolean Type When Saving DataFrame] Provided Support For Boolean Data Type In CarbonDataFrameWriter
1.Provided Support For Boolean Data Type In CarbonDataFrameWriter
2.Test Cases are Added For Same
This closes #1491
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/91355ef7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/91355ef7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/91355ef7
Branch: refs/heads/fgdatamap
Commit: 91355ef7cb3147537eacd11c95518495417eab82
Parents: d74251f
Author: anubhav100 <an...@knoldus.in>
Authored: Mon Nov 13 13:33:15 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Nov 18 01:37:02 2017 +0800
----------------------------------------------------------------------
.../testsuite/dataload/TestLoadDataFrame.scala | 27 ++++++++++++++++++--
.../spark/sql/CarbonDataFrameWriter.scala | 1 +
2 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91355ef7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index f2ea45e..3399740 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -20,7 +20,7 @@ package org.apache.carbondata.spark.testsuite.dataload
import java.math.BigDecimal
import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.types.{DecimalType, DoubleType, StringType, StructField, StructType}
+import org.apache.spark.sql.types._
import org.apache.spark.sql.{DataFrame, Row, SaveMode}
import org.scalatest.BeforeAndAfterAll
@@ -28,6 +28,7 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
var df: DataFrame = _
var dataFrame: DataFrame = _
var df2: DataFrame = _
+ var booldf:DataFrame = _
def buildTestData() = {
@@ -49,6 +50,15 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
df2 = sqlContext.sparkContext.parallelize(1 to 1000)
.map(x => ("key_" + x, "str_" + x, x, x * 2, x * 3))
.toDF("c1", "c2", "c3", "c4", "c5")
+
+ val boolrdd = sqlContext.sparkContext.parallelize(
+ Row("anubhav",true) ::
+ Row("prince",false) :: Nil)
+
+ val boolSchema = StructType(
+ StructField("name", StringType, nullable = false) ::
+ StructField("isCarbonEmployee",BooleanType,nullable = false)::Nil)
+ booldf = sqlContext.createDataFrame(boolrdd,boolSchema)
}
def dropTable() = {
@@ -61,6 +71,8 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS carbon7")
sql("DROP TABLE IF EXISTS carbon8")
sql("DROP TABLE IF EXISTS carbon9")
+ sql("DROP TABLE IF EXISTS carbon10")
+
}
@@ -70,7 +82,18 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
buildTestData
}
-
+test("test the boolean data type"){
+ booldf.write
+ .format("carbondata")
+ .option("tableName", "carbon10")
+ .option("tempCSV", "true")
+ .option("compress", "true")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("SELECT * FROM CARBON10"),
+ Seq(Row("anubhav", true), Row("prince", false)))
+}
test("test load dataframe with saving compressed csv files") {
// save dataframe to carbon file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/91355ef7/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index b74576d..89b618f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -153,6 +153,7 @@ class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame) {
case TimestampType => CarbonType.TIMESTAMP.getName
case DateType => CarbonType.DATE.getName
case decimal: DecimalType => s"decimal(${decimal.precision}, ${decimal.scale})"
+ case BooleanType => CarbonType.BOOLEAN.getName
case other => sys.error(s"unsupported type: $other")
}
}
[15/28] carbondata git commit: [CARBONDATA-1741] Remove AKSK in log
when saving to S3
Posted by ja...@apache.org.
[CARBONDATA-1741] Remove AKSK in log when saving to S3
This closes #1511
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/75ec79e3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/75ec79e3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/75ec79e3
Branch: refs/heads/fgdatamap
Commit: 75ec79e3f7ddbbfcddb1fbfe1413aae2ee13b6e2
Parents: 52bf7c8
Author: Jacky Li <ja...@qq.com>
Authored: Fri Nov 17 14:57:15 2017 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Fri Nov 17 16:22:11 2017 +0800
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 19 ++++++++++++++++++-
.../scala/org/apache/spark/util/FileUtils.scala | 8 +++++---
2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75ec79e3/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 1e402d0..3c177dc 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -764,6 +764,23 @@ public final class CarbonUtil {
lowerPath.startsWith(CarbonCommonConstants.S3A_PREFIX);
}
+ public static String removeAKSK(String filePath) {
+ if (null == filePath) {
+ return "";
+ }
+ String lowerPath = filePath.toLowerCase(Locale.getDefault());
+ if (lowerPath.startsWith(CarbonCommonConstants.S3N_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.S3A_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.S3_PREFIX)) {
+ int prefixLength = filePath.indexOf(":", 0) + 3;
+ int pathOffset = filePath.indexOf("@");
+ if (pathOffset > 0) {
+ return filePath.substring(0, prefixLength) + filePath.substring(pathOffset + 1);
+ }
+ }
+ return filePath;
+ }
+
/**
* This method will check the existence of a file at a given path
*/
@@ -774,7 +791,7 @@ public final class CarbonUtil {
return true;
}
} catch (IOException e) {
- LOGGER.error("@@@@@@ File not found at a given location @@@@@@ : " + fileName);
+ LOGGER.error("@@@@@@ File not found at a given location @@@@@@ : " + removeAKSK(fileName));
}
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75ec79e3/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
index 7345d54..46311db 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
@@ -21,6 +21,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.filesystem.CarbonFile
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.processing.exception.DataLoadingException
object FileUtils {
@@ -38,11 +39,11 @@ object FileUtils {
val fileName = carbonFile.getName
if (carbonFile.getSize == 0) {
LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- .warn(s"skip empty input file: $path")
+ .warn(s"skip empty input file: ${CarbonUtil.removeAKSK(path)}")
} else if (fileName.startsWith(CarbonCommonConstants.UNDERSCORE) ||
fileName.startsWith(CarbonCommonConstants.POINT)) {
LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- .warn(s"skip invisible input file: $path")
+ .warn(s"skip invisible input file: ${CarbonUtil.removeAKSK(path)}")
} else {
stringBuild.append(path.replace('\\', '/')).append(CarbonCommonConstants.COMMA)
}
@@ -63,7 +64,8 @@ object FileUtils {
val fileType = FileFactory.getFileType(filePaths(i))
val carbonFile = FileFactory.getCarbonFile(filePaths(i), fileType)
if (!carbonFile.exists()) {
- throw new DataLoadingException(s"The input file does not exist: ${filePaths(i)}" )
+ throw new DataLoadingException(
+ s"The input file does not exist: ${CarbonUtil.removeAKSK(filePaths(i))}" )
}
getPathsFromCarbonFile(carbonFile, stringBuild)
}
[18/28] carbondata git commit: [CARBONDATA-1706] Making index merge
DDL insensitive to the property
Posted by ja...@apache.org.
[CARBONDATA-1706] Making index merge DDL insensitive to the property
(1) Segment index merge DDL will be insensitive to the property. Even if the property is set to false, DDL should be able to merge index files
(2) Compaction name is changed from SEGMENT_INDEX_COMPACTION to SEGMENT_INDEX
This closes #1494
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f22e6146
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f22e6146
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f22e6146
Branch: refs/heads/fgdatamap
Commit: f22e61460ab6c006617b0ec3c332de7f49c74883
Parents: 1d2af62
Author: dhatchayani <dh...@gmail.com>
Authored: Tue Nov 14 15:17:14 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Nov 17 21:39:22 2017 +0530
----------------------------------------------------------------------
.../sdv/generated/MergeIndexTestCase.scala | 2 +-
.../CarbonIndexFileMergeTestCase.scala | 25 ++++++++++++++++++-
.../apache/carbondata/spark/rdd/Compactor.scala | 3 ++-
.../carbondata/spark/util/CommonUtil.scala | 26 +++++++++++++++++---
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../AlterTableCompactionCommand.scala | 5 ++--
.../sql/execution/strategy/DDLStrategy.scala | 2 +-
7 files changed, 53 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f22e6146/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
index 2e26d7f..758c897 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
@@ -79,7 +79,7 @@ class MergeIndexTestCase extends QueryTest with BeforeAndAfterAll {
assert(getIndexFileCount("default", "carbon_automation_nonmerge", "1") == 2)
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT, "true")
- sql("ALTER TABLE carbon_automation_nonmerge COMPACT 'SEGMENT_INDEX_COMPACTION'").collect()
+ sql("ALTER TABLE carbon_automation_nonmerge COMPACT 'SEGMENT_INDEX'").collect()
assert(getIndexFileCount("default", "carbon_automation_nonmerge", "0") == 0)
assert(getIndexFileCount("default", "carbon_automation_nonmerge", "1") == 0)
checkAnswer(sql("""Select count(*) from carbon_automation_nonmerge"""), rows)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f22e6146/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
index 110557c..f06994c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
@@ -92,7 +92,30 @@ class CarbonIndexFileMergeTestCase
assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT, "true")
- sql("ALTER TABLE nonindexmerge COMPACT 'SEGMENT_INDEX_COMPACTION'").collect()
+ sql("ALTER TABLE nonindexmerge COMPACT 'SEGMENT_INDEX'").collect()
+ assert(getIndexFileCount("default_nonindexmerge", "0") == 0)
+ assert(getIndexFileCount("default_nonindexmerge", "1") == 0)
+ checkAnswer(sql("""Select count(*) from nonindexmerge"""), rows)
+ }
+
+ test("Verify command of index merge without enabling property") {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT, "false")
+ sql("DROP TABLE IF EXISTS nonindexmerge")
+ sql(
+ """
+ | CREATE TABLE nonindexmerge(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE nonindexmerge OPTIONS('header'='false', " +
+ s"'GLOBAL_SORT_PARTITIONS'='100')")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE nonindexmerge OPTIONS('header'='false', " +
+ s"'GLOBAL_SORT_PARTITIONS'='100')")
+ val rows = sql("""Select count(*) from nonindexmerge""").collect()
+ assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
+ assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
+ sql("ALTER TABLE nonindexmerge COMPACT 'SEGMENT_INDEX'").collect()
assert(getIndexFileCount("default_nonindexmerge", "0") == 0)
assert(getIndexFileCount("default_nonindexmerge", "1") == 0)
checkAnswer(sql("""Select count(*) from nonindexmerge"""), rows)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f22e6146/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
index a787af2..7057816 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
@@ -113,7 +113,8 @@ object Compactor {
if (finalMergeStatus) {
val mergedLoadNumber = CarbonDataMergerUtil.getLoadNumberFromLoadName(mergedLoadName)
- CommonUtil.mergeIndexFiles(sc.sparkContext, Seq(mergedLoadNumber), storePath, carbonTable)
+ CommonUtil.mergeIndexFiles(
+ sc.sparkContext, Seq(mergedLoadNumber), storePath, carbonTable, false)
// trigger event for compaction
val alterTableCompactionPostEvent: AlterTableCompactionPostEvent =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f22e6146/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 6c0e802..f0b33f4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -845,13 +845,31 @@ object CommonUtil {
def mergeIndexFiles(sparkContext: SparkContext,
segmentIds: Seq[String],
tablePath: String,
- carbonTable: CarbonTable): Unit = {
- if (CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT,
- CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT).toBoolean) {
+ carbonTable: CarbonTable,
+ mergeIndexProperty: Boolean): Unit = {
+ if (mergeIndexProperty) {
new CarbonMergeFilesRDD(sparkContext, AbsoluteTableIdentifier.from(tablePath,
carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
segmentIds).collect()
+ } else {
+ try {
+ CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT).toBoolean
+ if (CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT,
+ CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT).toBoolean) {
+ new CarbonMergeFilesRDD(sparkContext, AbsoluteTableIdentifier.from(tablePath,
+ carbonTable.getDatabaseName, carbonTable.getFactTableName).getTablePath,
+ segmentIds).collect()
+ }
+ } catch {
+ case _: Exception =>
+ if (CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT.toBoolean) {
+ new CarbonMergeFilesRDD(sparkContext, AbsoluteTableIdentifier.from(tablePath,
+ carbonTable.getDatabaseName, carbonTable.getFactTableName).getTablePath,
+ segmentIds).collect()
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f22e6146/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index c12d2ef..e32c407 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -330,7 +330,7 @@ object CarbonDataRDDFactory {
loadDataFile(sqlContext, carbonLoadModel)
}
CommonUtil.mergeIndexFiles(sqlContext.sparkContext,
- Seq(carbonLoadModel.getSegmentId), storePath, carbonTable)
+ Seq(carbonLoadModel.getSegmentId), storePath, carbonTable, false)
val newStatusMap = scala.collection.mutable.Map.empty[String, SegmentStatus]
if (status.nonEmpty) {
status.foreach { eachLoadStatus =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f22e6146/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala
index 2f04feb..51275aa 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/AlterTableCompactionCommand.scala
@@ -113,8 +113,7 @@ case class AlterTableCompactionCommand(
carbonLoadModel.setLoadMetadataDetails(
alterTableModel.segmentUpdateStatusManager.get.getLoadMetadataDetails.toList.asJava)
}
- } else if (alterTableModel.compactionType.equalsIgnoreCase(
- CompactionType.SEGMENT_INDEX_COMPACTION.toString)) {
+ } else if (alterTableModel.compactionType.equalsIgnoreCase("segment_index")) {
compactionType = CompactionType.SEGMENT_INDEX_COMPACTION
} else {
compactionType = CompactionType.MINOR_COMPACTION
@@ -132,7 +131,7 @@ case class AlterTableCompactionCommand(
CommonUtil.mergeIndexFiles(sqlContext.sparkContext,
carbonLoadModel.getLoadMetadataDetails.asScala.map(_.getLoadName),
carbonLoadModel.getTablePath,
- carbonTable)
+ carbonTable, true)
return
}
// reading the start time of data load.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f22e6146/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index d6450c1..4e75547 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -85,7 +85,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
if (isCarbonTable) {
if (altertablemodel.compactionType.equalsIgnoreCase("minor") ||
altertablemodel.compactionType.equalsIgnoreCase("major") ||
- altertablemodel.compactionType.equalsIgnoreCase("SEGMENT_INDEX_COMPACTION")) {
+ altertablemodel.compactionType.equalsIgnoreCase("segment_index")) {
ExecutedCommandExec(alterTable) :: Nil
} else {
throw new MalformedCarbonCommandException(
[23/28] carbondata git commit: [CARBONDATA-1753][Streaming]Fix
missing 'org.scalatest.tools.Runner' issue when run test with streaming
module
Posted by ja...@apache.org.
[CARBONDATA-1753][Streaming]Fix missing 'org.scalatest.tools.Runner' issue when run test with streaming module
This closes #1519
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1b8d348c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1b8d348c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1b8d348c
Branch: refs/heads/fgdatamap
Commit: 1b8d348c041021038a5a542de1ef7da1855b514a
Parents: ee71610
Author: Zhang Zhichao <44...@qq.com>
Authored: Fri Nov 17 14:27:23 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Sat Nov 18 10:48:08 2017 +0800
----------------------------------------------------------------------
streaming/pom.xml | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1b8d348c/streaming/pom.xml
----------------------------------------------------------------------
diff --git a/streaming/pom.xml b/streaming/pom.xml
index d9dac75..713bf2d 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -28,7 +28,12 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>3.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <version>2.2.1</version>
<scope>test</scope>
</dependency>
</dependencies>
[07/28] carbondata git commit: [CARBONDATA-1732] Add S3 support in
FileFactory
Posted by ja...@apache.org.
[CARBONDATA-1732] Add S3 support in FileFactory
This closes #1504
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/733bb516
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/733bb516
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/733bb516
Branch: refs/heads/fgdatamap
Commit: 733bb516dc3fc4a1e2be02b6574c70aafa7d3b9d
Parents: 6551620
Author: Jacky Li <ja...@qq.com>
Authored: Thu Nov 16 17:27:21 2017 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Fri Nov 17 09:54:56 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 21 +++++++-------
.../core/datastore/impl/FileFactory.java | 30 +++++++++++++++++---
.../apache/carbondata/core/util/CarbonUtil.java | 11 ++++---
.../core/util/path/HDFSLeaseUtils.java | 1 +
.../carbondata/hadoop/util/SchemaReader.java | 1 +
.../spark/rdd/CarbonDataRDDFactory.scala | 20 +++++++++----
6 files changed, 61 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/733bb516/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index aeca19f..0a7dfdd 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -147,19 +147,21 @@ public final class CarbonCommonConstants {
* Load Folder Name
*/
public static final String LOAD_FOLDER = "Segment_";
- /**
- * HDFSURL_PREFIX
- */
+
public static final String HDFSURL_PREFIX = "hdfs://";
- /**
- * VIEWFSURL_PREFIX
- */
+
+ public static final String LOCAL_FILE_PREFIX = "file://";
+
public static final String VIEWFSURL_PREFIX = "viewfs://";
- /**
- * ALLUXIO_PREFIX
- */
public static final String ALLUXIOURL_PREFIX = "alluxio://";
+
+ public static final String S3_PREFIX = "s3://";
+
+ public static final String S3N_PREFIX = "s3n://";
+
+ public static final String S3A_PREFIX = "s3a://";
+
/**
* FS_DEFAULT_FS
*/
@@ -1261,7 +1263,6 @@ public final class CarbonCommonConstants {
public static final String MAJOR = "major";
- public static final String LOCAL_FILE_PREFIX = "file://";
@CarbonProperty
public static final String CARBON_CUSTOM_BLOCK_DISTRIBUTION = "carbon.custom.block.distribution";
public static final String CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT = "false";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/733bb516/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index e4e4ae2..57a48ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -73,6 +73,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
return new DFSFileHolderImpl();
default:
return new FileHolderImpl();
@@ -80,12 +81,17 @@ public final class FileFactory {
}
public static FileType getFileType(String path) {
- if (path.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) {
+ String lowerPath = path.toLowerCase();
+ if (lowerPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) {
return FileType.HDFS;
- } else if (path.startsWith(CarbonCommonConstants.ALLUXIOURL_PREFIX)) {
+ } else if (lowerPath.startsWith(CarbonCommonConstants.ALLUXIOURL_PREFIX)) {
return FileType.ALLUXIO;
- } else if (path.startsWith(CarbonCommonConstants.VIEWFSURL_PREFIX)) {
+ } else if (lowerPath.startsWith(CarbonCommonConstants.VIEWFSURL_PREFIX)) {
return FileType.VIEWFS;
+ } else if (lowerPath.startsWith(CarbonCommonConstants.S3N_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.S3A_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.S3_PREFIX)) {
+ return FileType.S3;
}
return FileType.LOCAL;
}
@@ -99,6 +105,7 @@ public final class FileFactory {
case LOCAL:
return new LocalCarbonFile(getUpdatedFilePath(path, fileType));
case HDFS:
+ case S3:
return new HDFSCarbonFile(path);
case ALLUXIO:
return new AlluxioCarbonFile(path);
@@ -134,6 +141,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
if (bufferSize == -1) {
@@ -176,6 +184,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
FSDataInputStream stream = fs.open(pt, bufferSize);
@@ -203,6 +212,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
return fs.create(pt, true);
@@ -222,6 +232,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
FSDataOutputStream stream = null;
@@ -255,6 +266,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
return fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
@@ -280,6 +292,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path path = new Path(filePath);
FileSystem fs = path.getFileSystem(configuration);
if (performFileCheck) {
@@ -314,6 +327,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path path = new Path(filePath);
FileSystem fs = path.getFileSystem(configuration);
return fs.exists(path);
@@ -332,6 +346,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path path = new Path(filePath);
FileSystem fs = path.getFileSystem(configuration);
return fs.createNewFile(path);
@@ -350,6 +365,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path path = new Path(filePath);
FileSystem fs = path.getFileSystem(configuration);
return fs.delete(path, true);
@@ -399,6 +415,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path path = new Path(filePath);
FileSystem fs = path.getFileSystem(configuration);
return fs.mkdirs(path);
@@ -428,6 +445,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
return fs.append(pt);
@@ -461,6 +479,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
fs.truncate(pt, newSize);
@@ -493,6 +512,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path path = new Path(filePath);
FileSystem fs = path.getFileSystem(configuration);
if (fs.createNewFile(path)) {
@@ -509,7 +529,7 @@ public final class FileFactory {
}
public enum FileType {
- LOCAL, HDFS, ALLUXIO, VIEWFS
+ LOCAL, HDFS, ALLUXIO, VIEWFS, S3
}
/**
@@ -526,6 +546,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
return filePath;
case LOCAL:
default:
@@ -574,6 +595,7 @@ public final class FileFactory {
case HDFS:
case ALLUXIO:
case VIEWFS:
+ case S3:
Path path = new Path(filePath);
FileSystem fs = path.getFileSystem(configuration);
return fs.getContentSummary(path).getLength();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/733bb516/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index e709df7..bdd7ba3 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -754,10 +754,13 @@ public final class CarbonUtil {
private static boolean checkIfPrefixExists(String path) {
final String lowerPath = path.toLowerCase();
- return lowerPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX) || lowerPath
- .startsWith(CarbonCommonConstants.VIEWFSURL_PREFIX) || lowerPath
- .startsWith(CarbonCommonConstants.LOCAL_FILE_PREFIX) || lowerPath
- .startsWith(CarbonCommonConstants.ALLUXIOURL_PREFIX);
+ return lowerPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.VIEWFSURL_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.LOCAL_FILE_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.ALLUXIOURL_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.S3N_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.S3_PREFIX) ||
+ lowerPath.startsWith(CarbonCommonConstants.S3A_PREFIX);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/733bb516/core/src/main/java/org/apache/carbondata/core/util/path/HDFSLeaseUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/HDFSLeaseUtils.java b/core/src/main/java/org/apache/carbondata/core/util/path/HDFSLeaseUtils.java
index c72c322..fcd1655 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/HDFSLeaseUtils.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/HDFSLeaseUtils.java
@@ -78,6 +78,7 @@ public class HDFSLeaseUtils {
switch (fileType) {
case ALLUXIO:
case HDFS:
+ case S3:
Path path = FileFactory.getPath(filePath);
FileSystem fs = FileFactory.getFileSystem(path);
return recoverLeaseOnFile(filePath, path, (DistributedFileSystem) fs);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/733bb516/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
index f1ce324..2e6abad 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
@@ -42,6 +42,7 @@ public class SchemaReader {
String schemaFilePath = carbonTablePath.getSchemaFilePath();
if (FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.LOCAL) ||
FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.HDFS) ||
+ FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.S3) ||
FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.VIEWFS)) {
String tableName = identifier.getCarbonTableIdentifier().getTableName();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/733bb516/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 7c5599b..1ca7456 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -487,8 +487,17 @@ object CarbonDataRDDFactory {
carbonTable.getCarbonTableIdentifier,
carbonLoadModel)
OperationListenerBus.getInstance.fireEvent(loadTablePostExecutionEvent, operationContext)
- updateTableStatus(status, carbonLoadModel, loadStatus, overwriteTable)
-
+ val done = updateTableStatus(status, carbonLoadModel, loadStatus, overwriteTable)
+ if (!done) {
+ CommonUtil.updateTableStatusForFailure(carbonLoadModel)
+ LOGGER.info("********starting clean up**********")
+ CarbonLoaderUtil.deleteSegment(carbonLoadModel, carbonLoadModel.getSegmentId.toInt)
+ LOGGER.info("********clean up done**********")
+ LOGGER.audit("Data load is failed for " +
+ s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
+ LOGGER.error("Data load failed due to failure in table status updation.")
+ throw new Exception("Data load failed due to failure in table status updation.")
+ }
if (SegmentStatus.LOAD_PARTIAL_SUCCESS == loadStatus) {
LOGGER.audit("Data load is partially successful for " +
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
@@ -738,7 +747,7 @@ object CarbonDataRDDFactory {
carbonLoadModel: CarbonLoadModel,
loadStatus: SegmentStatus,
overwriteTable: Boolean
- ): Unit = {
+ ): Boolean = {
val metadataDetails = if (status != null && status(0) != null) {
status(0)._2._1
} else {
@@ -749,9 +758,9 @@ object CarbonDataRDDFactory {
loadStatus,
carbonLoadModel.getFactTimeStamp,
true)
- val success = CarbonLoaderUtil.recordLoadMetadata(metadataDetails, carbonLoadModel, false,
+ val done = CarbonLoaderUtil.recordLoadMetadata(metadataDetails, carbonLoadModel, false,
overwriteTable)
- if (!success) {
+ if (!done) {
val errorMessage = "Dataload failed due to failure in table status updation."
LOGGER.audit("Data load is failed for " +
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
@@ -761,6 +770,7 @@ object CarbonDataRDDFactory {
// TODO : Handle it
LOGGER.info("********Database updated**********")
}
+ done
}
[20/28] carbondata git commit: [CARBONDATA-1764] Fix issue of when
create table with short data type
Posted by ja...@apache.org.
[CARBONDATA-1764] Fix issue of when create table with short data type
Fix issue of when create table with short data type
This closes #1526
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d74251fa
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d74251fa
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d74251fa
Branch: refs/heads/fgdatamap
Commit: d74251fa35debc5703fc8cb128eef3f58d9ab59f
Parents: 589f126
Author: xubo245 <60...@qq.com>
Authored: Fri Nov 17 23:23:31 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Nov 18 01:32:34 2017 +0800
----------------------------------------------------------------------
.../aggquery/IntegerDataTypeTestCase.scala | 21 ++++++++++++-
.../spark/util/DataTypeConverterUtilSuite.scala | 33 ++++++++++++++++++++
.../spark/util/DataTypeConverterUtil.scala | 1 +
3 files changed, 54 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d74251fa/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
index dc4dc3a..4f9d09d 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
@@ -32,6 +32,7 @@ class IntegerDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
sql("DROP TABLE IF EXISTS integertypetableAgg")
+ sql("DROP TABLE IF EXISTS short_table")
sql("CREATE TABLE integertypetableAgg (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE integertypetableAgg OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
}
@@ -141,7 +142,25 @@ class IntegerDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
| DROP TABLE short_int_target_table
""".stripMargin)
}
-
+
+ test("Create a table that contains short data type") {
+ sql("CREATE TABLE if not exists short_table(col1 short, col2 BOOLEAN) STORED BY 'carbondata'")
+
+ sql("insert into short_table values(1,true)")
+ sql("insert into short_table values(11,false)")
+ sql("insert into short_table values(211,false)")
+ sql("insert into short_table values(3111,true)")
+ sql("insert into short_table values(31111,false)")
+ sql("insert into short_table values(411111,false)")
+ sql("insert into short_table values(5111111,true)")
+
+ checkAnswer(
+ sql("select count(*) from short_table"),
+ Row(7)
+ )
+ sql("DROP TABLE IF EXISTS short_table")
+ }
+
override def afterAll {
sql("drop table if exists integertypetableAgg")
CarbonProperties.getInstance().addProperty(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d74251fa/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/util/DataTypeConverterUtilSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/util/DataTypeConverterUtilSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/util/DataTypeConverterUtilSuite.scala
new file mode 100644
index 0000000..0dd7b23
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/util/DataTypeConverterUtilSuite.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.util
+
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.spark.sql.test.util.QueryTest
+
+import org.apache.carbondata.core.metadata.datatype.DataTypes
+
+/**
+ * test [[DataTypeConverterUtil]]
+ */
+class DataTypeConverterUtilSuite extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
+ test("test short convert") {
+ assert(DataTypeConverterUtil.convertToCarbonType("short") == DataTypes.SHORT)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d74251fa/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 38657ac..0969fdf 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -32,6 +32,7 @@ object DataTypeConverterUtil {
case "integer" => DataTypes.INT
case "tinyint" => DataTypes.SHORT
case "smallint" => DataTypes.SHORT
+ case "short" => DataTypes.SHORT
case "long" => DataTypes.LONG
case "bigint" => DataTypes.LONG
case "numeric" => DataTypes.DOUBLE
[28/28] carbondata git commit: [CARBONDATA-1544][Datamap] Datamap
FineGrain implementation
Posted by ja...@apache.org.
[CARBONDATA-1544][Datamap] Datamap FineGrain implementation
Implemented interfaces for FG datamap and integrated to filterscanner to use the pruned bitset from FG datamap.
FG Query flow as follows.
1.The user can add FG datamap to any table and implement there interfaces.
2. Any filter query which hits the table with datamap will call prune method of FGdatamap.
3. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the information of block, blocklet, page and rowids information as well.
4. The pruned blocklets are internally wriitten to file and returns only the block , blocklet and filepath information as part of Splits.
5. Based on the splits scanrdd schedule the tasks.
6. In filterscanner we check the datamapwriterpath from split and reNoteads the bitset if exists. And pass this bitset as input to it.
This closes #1471
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/36c34cd0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/36c34cd0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/36c34cd0
Branch: refs/heads/fgdatamap
Commit: 36c34cd02893bd59018d7cab89778c314c3b119e
Parents: 2943108
Author: ravipesala <ra...@gmail.com>
Authored: Wed Nov 15 19:48:40 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Nov 18 23:25:19 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/datamap/DataMapMeta.java | 8 +-
.../core/datamap/DataMapStoreManager.java | 30 +-
.../carbondata/core/datamap/DataMapType.java | 21 +
.../carbondata/core/datamap/TableDataMap.java | 31 +-
.../core/datamap/dev/AbstractDataMapWriter.java | 110 +++++
.../core/datamap/dev/BlockletSerializer.java | 57 +++
.../carbondata/core/datamap/dev/DataMap.java | 4 +-
.../core/datamap/dev/DataMapFactory.java | 14 +-
.../core/datamap/dev/DataMapWriter.java | 57 ---
.../cgdatamap/AbstractCoarseGrainDataMap.java | 24 +
.../AbstractCoarseGrainDataMapFactory.java | 34 ++
.../dev/fgdatamap/AbstractFineGrainDataMap.java | 24 +
.../AbstractFineGrainDataMapFactory.java | 38 ++
.../carbondata/core/datastore/DataRefNode.java | 7 +
.../core/datastore/block/TableBlockInfo.java | 10 +
.../impl/btree/AbstractBTreeLeafNode.java | 5 +
.../datastore/impl/btree/BTreeNonLeafNode.java | 5 +
.../carbondata/core/indexstore/Blocklet.java | 30 +-
.../core/indexstore/BlockletDetailsFetcher.java | 8 +
.../core/indexstore/ExtendedBlocklet.java | 19 +-
.../core/indexstore/FineGrainBlocklet.java | 120 +++++
.../blockletindex/BlockletDataMap.java | 11 +-
.../blockletindex/BlockletDataMapFactory.java | 62 ++-
.../BlockletDataRefNodeWrapper.java | 27 +-
.../indexstore/blockletindex/IndexWrapper.java | 18 +
.../core/indexstore/schema/FilterType.java | 24 -
.../executer/ExcludeFilterExecuterImpl.java | 3 +
.../executer/IncludeFilterExecuterImpl.java | 3 +
.../core/scan/processor/BlocksChunkHolder.java | 5 -
.../core/scan/scanner/impl/FilterScanner.java | 2 +
.../apache/carbondata/core/util/CarbonUtil.java | 97 +++++
.../datamap/examples/MinMaxDataMap.java | 20 +-
.../datamap/examples/MinMaxDataMapFactory.java | 49 ++-
.../datamap/examples/MinMaxDataWriter.java | 36 +-
.../examples/MinMaxIndexBlockDetails.java | 13 -
.../carbondata/hadoop/CarbonInputFormat.java | 2 +-
.../carbondata/hadoop/CarbonInputSplit.java | 20 +-
.../hadoop/api/CarbonTableInputFormat.java | 23 +-
.../testsuite/datamap/CGDataMapTestCase.scala | 357 +++++++++++++++
.../testsuite/datamap/DataMapWriterSuite.scala | 49 ++-
.../testsuite/datamap/FGDataMapTestCase.scala | 436 +++++++++++++++++++
.../carbondata/spark/rdd/CarbonScanRDD.scala | 6 +-
.../datamap/DataMapWriterListener.java | 57 ++-
.../store/CarbonFactDataHandlerModel.java | 10 +-
.../store/writer/AbstractFactDataWriter.java | 126 +-----
45 files changed, 1731 insertions(+), 381 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
index 7746acf..dd15ccb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
@@ -19,15 +19,15 @@ package org.apache.carbondata.core.datamap;
import java.util.List;
-import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
public class DataMapMeta {
private List<String> indexedColumns;
- private FilterType optimizedOperation;
+ private List<ExpressionType> optimizedOperation;
- public DataMapMeta(List<String> indexedColumns, FilterType optimizedOperation) {
+ public DataMapMeta(List<String> indexedColumns, List<ExpressionType> optimizedOperation) {
this.indexedColumns = indexedColumns;
this.optimizedOperation = optimizedOperation;
}
@@ -36,7 +36,7 @@ public class DataMapMeta {
return indexedColumns;
}
- public FilterType getOptimizedOperation() {
+ public List<ExpressionType> getOptimizedOperation() {
return optimizedOperation;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 90e5fff..8d80b4d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -56,7 +56,22 @@ public final class DataMapStoreManager {
}
public List<TableDataMap> getAllDataMap(AbsoluteTableIdentifier identifier) {
- return allDataMaps.get(identifier.uniqueName());
+ return allDataMaps.get(identifier.getCarbonTableIdentifier().getTableUniqueName());
+ }
+
+ // TODO its a temporary method till chooser is implemented
+ public TableDataMap chooseDataMap(AbsoluteTableIdentifier identifier) {
+ List<TableDataMap> tableDataMaps = getAllDataMap(identifier);
+ if (tableDataMaps != null && tableDataMaps.size() > 0) {
+ for (TableDataMap dataMap: tableDataMaps) {
+ if (!dataMap.getDataMapName().equalsIgnoreCase(BlockletDataMap.NAME)) {
+ return dataMap;
+ }
+ }
+ return tableDataMaps.get(0);
+ } else {
+ return getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName());
+ }
}
/**
@@ -68,7 +83,7 @@ public final class DataMapStoreManager {
*/
public TableDataMap getDataMap(AbsoluteTableIdentifier identifier,
String dataMapName, String factoryClass) {
- String table = identifier.uniqueName();
+ String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
List<TableDataMap> tableDataMaps = allDataMaps.get(table);
TableDataMap dataMap;
if (tableDataMaps == null) {
@@ -96,7 +111,7 @@ public final class DataMapStoreManager {
*/
public TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
String factoryClassName, String dataMapName) {
- String table = identifier.uniqueName();
+ String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
// Just update the segmentRefreshMap with the table if not added.
getTableSegmentRefresher(identifier);
List<TableDataMap> tableDataMaps = allDataMaps.get(table);
@@ -149,7 +164,9 @@ public final class DataMapStoreManager {
* @param identifier Table identifier
*/
public void clearDataMaps(AbsoluteTableIdentifier identifier) {
- List<TableDataMap> tableDataMaps = allDataMaps.get(identifier.uniqueName());
+ String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName();
+ List<TableDataMap> tableDataMaps =
+ allDataMaps.get(tableUniqueName);
segmentRefreshMap.remove(identifier.uniqueName());
if (tableDataMaps != null) {
for (TableDataMap tableDataMap: tableDataMaps) {
@@ -158,7 +175,7 @@ public final class DataMapStoreManager {
break;
}
}
- allDataMaps.remove(identifier.uniqueName());
+ allDataMaps.remove(tableUniqueName);
}
}
@@ -167,7 +184,8 @@ public final class DataMapStoreManager {
* @param identifier Table identifier
*/
public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) {
- List<TableDataMap> tableDataMaps = allDataMaps.get(identifier.uniqueName());
+ List<TableDataMap> tableDataMaps =
+ allDataMaps.get(identifier.getCarbonTableIdentifier().getTableUniqueName());
if (tableDataMaps != null) {
int i = 0;
for (TableDataMap tableDataMap: tableDataMaps) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
new file mode 100644
index 0000000..bf812b3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap;
+
+public enum DataMapType {
+ CG,FG;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 07d8eb7..1704e34 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -20,12 +20,16 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -75,10 +79,15 @@ public final class TableDataMap implements OperationEventListener {
SegmentProperties segmentProperties;
for (String segmentId : segmentIds) {
List<Blocklet> pruneBlocklets = new ArrayList<>();
- List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
- segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId);
- for (DataMap dataMap : dataMaps) {
- pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties));
+ // if filter is not passed then return all the blocklets
+ if (filterExp == null) {
+ pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segmentId);
+ } else {
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
+ segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId);
+ for (DataMap dataMap : dataMaps) {
+ pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties));
+ }
}
blocklets.addAll(addSegmentId(blockletDetailsFetcher
.getExtendedBlocklets(pruneBlocklets, segmentId), segmentId));
@@ -133,9 +142,21 @@ public final class TableDataMap implements OperationEventListener {
blocklets.addAll(dataMap.prune(filterExp,
segmentPropertiesFetcher.getSegmentProperties(distributable.getSegmentId())));
}
- for (Blocklet blocklet: blocklets) {
+ BlockletSerializer serializer = new BlockletSerializer();
+ String writePath =
+ identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + dataMapName;
+ if (dataMapFactory.getDataMapType() == DataMapType.FG) {
+ FileFactory.mkdirs(writePath, FileFactory.getFileType(writePath));
+ }
+ for (Blocklet blocklet : blocklets) {
ExtendedBlocklet detailedBlocklet =
blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegmentId());
+ if (dataMapFactory.getDataMapType() == DataMapType.FG) {
+ String blockletwritePath =
+ writePath + CarbonCommonConstants.FILE_SEPARATOR + System.nanoTime();
+ detailedBlocklet.setDataMapWriterPath(blockletwritePath);
+ serializer.serializeBlocklet((FineGrainBlocklet) blocklet, blockletwritePath);
+ }
detailedBlocklet.setSegmentId(distributable.getSegmentId());
detailedBlocklets.add(detailedBlocklet);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
new file mode 100644
index 0000000..bcc9bad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+/**
+ * Data Map writer
+ */
+public abstract class AbstractDataMapWriter {
+
+ protected AbsoluteTableIdentifier identifier;
+
+ protected String segmentId;
+
+ protected String writeDirectoryPath;
+
+ public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String writeDirectoryPath) {
+ this.identifier = identifier;
+ this.segmentId = segmentId;
+ this.writeDirectoryPath = writeDirectoryPath;
+ }
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ public abstract void onBlockStart(String blockId);
+
+ /**
+ * End of block notification
+ */
+ public abstract void onBlockEnd(String blockId);
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ public abstract void onBlockletStart(int blockletId);
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ public abstract void onBlockletEnd(int blockletId);
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ public abstract void onPageAdded(int blockletId, int pageId, ColumnPage[] pages);
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ public abstract void finish() throws IOException;
+
+ /**
+ * It copies the file from temp folder to actual folder
+ *
+ * @param dataMapFile
+ * @throws IOException
+ */
+ protected void commitFile(String dataMapFile) throws IOException {
+ if (!dataMapFile.startsWith(writeDirectoryPath)) {
+ throw new UnsupportedOperationException(
+ "Datamap file " + dataMapFile + " is not written in provided directory path "
+ + writeDirectoryPath);
+ }
+ String dataMapFileName =
+ dataMapFile.substring(writeDirectoryPath.length(), dataMapFile.length());
+ String carbonFilePath = dataMapFileName.substring(0, dataMapFileName.lastIndexOf("/"));
+ String segmentPath = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
+ if (carbonFilePath.length() > 0) {
+ carbonFilePath = segmentPath + carbonFilePath;
+ FileFactory.mkdirs(carbonFilePath, FileFactory.getFileType(carbonFilePath));
+ } else {
+ carbonFilePath = segmentPath;
+ }
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(dataMapFile, carbonFilePath, 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
new file mode 100644
index 0000000..3d4c717
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
+
+public class BlockletSerializer {
+
+ /**
+ * Serialize and write blocklet to the file.
+ * @param grainBlocklet
+ * @param writePath
+ * @throws IOException
+ */
+ public void serializeBlocklet(FineGrainBlocklet grainBlocklet, String writePath)
+ throws IOException {
+ DataOutputStream dataOutputStream =
+ FileFactory.getDataOutputStream(writePath, FileFactory.getFileType(writePath));
+ grainBlocklet.write(dataOutputStream);
+ dataOutputStream.close();
+ }
+
+ /**
+ * Read data from filepath and deserialize blocklet.
+ * @param writePath
+ * @return
+ * @throws IOException
+ */
+ public FineGrainBlocklet deserializeBlocklet(String writePath) throws IOException {
+ DataInputStream inputStream =
+ FileFactory.getDataInputStream(writePath, FileFactory.getFileType(writePath));
+ FineGrainBlocklet blocklet = new FineGrainBlocklet();
+ blocklet.readFields(inputStream);
+ inputStream.close();
+ return blocklet;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index 3731922..0e705bf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
/**
* Datamap is an entity which can store and retrieve index data.
*/
-public interface DataMap {
+public interface DataMap<T extends Blocklet> {
/**
* It is called to load the data map to memory or to initialize it.
@@ -41,7 +41,7 @@ public interface DataMap {
* @param filterExp
* @return
*/
- List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties);
+ List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties);
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index f5a7404..e900f8a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -21,13 +21,14 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.events.Event;
/**
* Interface for datamap factory, it is responsible for creating the datamap.
*/
-public interface DataMapFactory {
+public interface DataMapFactory<T extends DataMap> {
/**
* Initialization of Datamap factory with the identifier and datamap name
@@ -37,17 +38,17 @@ public interface DataMapFactory {
/**
* Return a new write for this datamap
*/
- DataMapWriter createWriter(String segmentId);
+ AbstractDataMapWriter createWriter(String segmentId, String writeDirectoryPath);
/**
* Get the datamap for segmentid
*/
- List<DataMap> getDataMaps(String segmentId) throws IOException;
+ List<T> getDataMaps(String segmentId) throws IOException;
/**
* Get datamaps for distributable object.
*/
- List<DataMap> getDataMaps(DataMapDistributable distributable) throws IOException;
+ List<T> getDataMaps(DataMapDistributable distributable) throws IOException;
/**
* Get all distributable objects of a segmentid
@@ -75,4 +76,9 @@ public interface DataMapFactory {
* Return metadata of this datamap
*/
DataMapMeta getMeta();
+
+ /**
+ * Type of datamap whether it is FG or CG
+ */
+ DataMapType getDataMapType();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
deleted file mode 100644
index 413eaa5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev;
-
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-
-/**
- * Data Map writer
- */
-public interface DataMapWriter {
-
- /**
- * Start of new block notification.
- * @param blockId file name of the carbondata file
- */
- void onBlockStart(String blockId, String blockPath);
-
- /**
- * End of block notification
- */
- void onBlockEnd(String blockId);
-
- /**
- * Start of new blocklet notification.
- * @param blockletId sequence number of blocklet in the block
- */
- void onBlockletStart(int blockletId);
-
- /**
- * End of blocklet notification
- * @param blockletId sequence number of blocklet in the block
- */
- void onBlockletEnd(int blockletId);
- /**
- * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
- * DataMapMeta returned in DataMapFactory.
- *
- * Implementation should copy the content of `pages` as needed, because `pages` memory
- * may be freed after this method returns, if using unsafe column page.
- */
- void onPageAdded(int blockletId, int pageId, ColumnPage[] pages);
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
new file mode 100644
index 0000000..d79d0c6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.indexstore.Blocklet;
+
+public abstract class AbstractCoarseGrainDataMap implements DataMap<Blocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
new file mode 100644
index 0000000..9789992
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+
+/**
+ * 1. Any filter query which hits the table with datamap will call prune method of CGdatamap.
+ * 2. The prune method of CGDatamap return list Blocklet , these blocklets contain the
+ * information of block and blocklet.
+ * 3. Based on the splits scanrdd schedule the tasks.
+ */
+public abstract class AbstractCoarseGrainDataMapFactory
+ implements DataMapFactory<AbstractCoarseGrainDataMap> {
+
+ @Override public DataMapType getDataMapType() {
+ return DataMapType.CG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
new file mode 100644
index 0000000..310fb3b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
+
+public abstract class AbstractFineGrainDataMap implements DataMap<FineGrainBlocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
new file mode 100644
index 0000000..1ca7fc3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+
+/**
+ * 1. Any filter query which hits the table with datamap will call prune method of FGdatamap.
+ * 2. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the
+ * information of block, blocklet, page and rowids information as well.
+ * 3. The pruned blocklets are internally wriitten to file and returns only the block ,
+ * blocklet and filepath information as part of Splits.
+ * 4. Based on the splits scanrdd schedule the tasks.
+ * 5. In filterscanner we check the datamapwriterpath from split and reNoteads the
+ * bitset if exists. And pass this bitset as input to it.
+ */
+public abstract class AbstractFineGrainDataMapFactory
+ implements DataMapFactory<AbstractFineGrainDataMap> {
+
+ @Override public DataMapType getDataMapType() {
+ return DataMapType.FG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 8914196..4ceb762 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* Interface data block reference
@@ -124,6 +125,12 @@ public interface DataRefNode {
BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache();
/**
+ * Return the indexed data if it has any from disk which was stored by FG datamap.
+ * @return
+ */
+ BitSetGroup getIndexedData();
+
+ /**
* number of pages in blocklet
* @return
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index 910c9bb..a1ef45f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -85,6 +85,8 @@ public class TableBlockInfo implements Distributable, Serializable {
private BlockletDetailInfo detailInfo;
+ private String dataMapWriterPath;
+
public TableBlockInfo(String filePath, long blockOffset, String segmentId, String[] locations,
long blockLength, ColumnarFormatVersion version, String[] deletedDeltaFilePath) {
this.filePath = FileFactory.getUpdatedFilePath(filePath);
@@ -356,4 +358,12 @@ public class TableBlockInfo implements Distributable, Serializable {
public void setDetailInfo(BlockletDetailInfo detailInfo) {
this.detailInfo = detailInfo;
}
+
+ public String getDataMapWriterPath() {
+ return dataMapWriterPath;
+ }
+
+ public void setDataMapWriterPath(String dataMapWriterPath) {
+ this.dataMapWriterPath = dataMapWriterPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
index dfd35bc..7a68423 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* Non leaf node abstract class
@@ -239,4 +240,8 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
return deleteDeltaDataCache;
}
+
+ @Override public BitSetGroup getIndexedData() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
index 01c0177..d78015b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* No leaf node of a b+tree class which will keep the matadata(start key) of the
@@ -245,6 +246,10 @@ public class BTreeNonLeafNode implements BTreeNode {
return deleteDeltaDataCache;
}
+ public BitSetGroup getIndexedData() {
+ return null;
+ }
+
/**
* number of pages in blocklet
* @return
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
index d84f3f6..c731e07 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
@@ -16,28 +16,46 @@
*/
package org.apache.carbondata.core.indexstore;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.io.Serializable;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+
/**
* Blocklet
*/
-public class Blocklet implements Serializable {
+public class Blocklet implements Writable,Serializable {
- private String path;
+ private String blockId;
private String blockletId;
- public Blocklet(String path, String blockletId) {
- this.path = path;
+ public Blocklet(String blockId, String blockletId) {
+ this.blockId = blockId;
this.blockletId = blockletId;
}
- public String getPath() {
- return path;
+ // For serialization purpose
+ public Blocklet() {
}
public String getBlockletId() {
return blockletId;
}
+ public String getBlockId() {
+ return blockId;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeUTF(blockId);
+ out.writeUTF(blockletId);
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ blockId = in.readUTF();
+ blockletId = in.readUTF();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
index 21ecba1..a493c06 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
@@ -44,4 +44,12 @@ public interface BlockletDetailsFetcher {
* @throws IOException
*/
ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, String segmentId) throws IOException;
+
+ /**
+ * Get all the blocklets in a segment
+ *
+ * @param segmentId
+ * @return
+ */
+ List<Blocklet> getAllBlocklets(String segmentId) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
index e0cfefb..081da53 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
@@ -38,8 +38,13 @@ public class ExtendedBlocklet extends Blocklet {
private String[] location;
+ private String path;
+
+ private String dataMapWriterPath;
+
public ExtendedBlocklet(String path, String blockletId) {
super(path, blockletId);
+ this.path = path;
}
public BlockletDetailInfo getDetailInfo() {
@@ -56,7 +61,7 @@ public class ExtendedBlocklet extends Blocklet {
* @throws IOException
*/
public void updateLocations() throws IOException {
- Path path = new Path(getPath());
+ Path path = new Path(this.path);
FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
LocatedFileStatus fileStatus = iter.next();
@@ -79,4 +84,16 @@ public class ExtendedBlocklet extends Blocklet {
public void setSegmentId(String segmentId) {
this.segmentId = segmentId;
}
+
+ public String getPath() {
+ return path;
+ }
+
+ public String getDataMapWriterPath() {
+ return dataMapWriterPath;
+ }
+
+ public void setDataMapWriterPath(String dataMapWriterPath) {
+ this.dataMapWriterPath = dataMapWriterPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
new file mode 100644
index 0000000..266120e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+/**
+ * FineGrainBlocklet
+ */
+public class FineGrainBlocklet extends Blocklet implements Serializable {
+
+ private List<Page> pages;
+
+ public FineGrainBlocklet(String blockId, String blockletId, List<Page> pages) {
+ super(blockId, blockletId);
+ this.pages = pages;
+ }
+
+ // For serialization purpose
+ public FineGrainBlocklet() {
+
+ }
+
+ public List<Page> getPages() {
+ return pages;
+ }
+
+ public static class Page implements Writable,Serializable {
+
+ private int pageId;
+
+ private int[] rowId;
+
+ public BitSet getBitSet() {
+ BitSet bitSet =
+ new BitSet(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
+ for (int row : rowId) {
+ bitSet.set(row);
+ }
+ return bitSet;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeInt(pageId);
+ out.writeInt(rowId.length);
+ for (int i = 0; i < rowId.length; i++) {
+ out.writeInt(rowId[i]);
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ pageId = in.readInt();
+ int length = in.readInt();
+ rowId = new int[length];
+ for (int i = 0; i < length; i++) {
+ rowId[i] = in.readInt();
+ }
+ }
+
+ public void setPageId(int pageId) {
+ this.pageId = pageId;
+ }
+
+ public void setRowId(int[] rowId) {
+ this.rowId = rowId;
+ }
+ }
+
+ public BitSetGroup getBitSetGroup(int numberOfPages) {
+ BitSetGroup bitSetGroup = new BitSetGroup(numberOfPages);
+ for (int i = 0; i < pages.size(); i++) {
+ bitSetGroup.setBitSet(pages.get(i).getBitSet(), pages.get(i).pageId);
+ }
+ return bitSetGroup;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ super.write(out);
+ int size = pages.size();
+ out.writeInt(size);
+ for (Page page : pages) {
+ page.write(out);
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ pages = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ Page page = new Page();
+ page.readFields(in);
+ pages.add(page);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 54016ad..a49e6f6 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cacheable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -65,7 +65,7 @@ import org.apache.carbondata.core.util.DataTypeUtil;
/**
* Datamap implementation for blocklet.
*/
-public class BlockletDataMap implements DataMap, Cacheable {
+public class BlockletDataMap extends AbstractCoarseGrainDataMap implements Cacheable {
private static final LogService LOGGER =
LogServiceFactory.getLogService(BlockletDataMap.class.getName());
@@ -315,7 +315,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
@Override
- public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties) {
+ public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segProps) {
// getting the start and end index key based on filter for hitting the
// selected block reference nodes based on filter resolver tree.
@@ -374,7 +374,6 @@ public class BlockletDataMap implements DataMap, Cacheable {
startIndex++;
}
}
-
return blocklets;
}
@@ -553,4 +552,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
}
+ public SegmentProperties getSegmentProperties() {
+ return segmentProperties;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 61e5ceb..48f8e05 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -27,9 +27,10 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -39,9 +40,6 @@ import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.DataFileFooterConverter;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -53,7 +51,8 @@ import org.apache.hadoop.fs.RemoteIterator;
/**
* Table map for blocklet
*/
-public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFetcher,
+public class BlockletDataMapFactory extends AbstractCoarseGrainDataMapFactory
+ implements BlockletDetailsFetcher,
SegmentPropertiesFetcher {
private AbsoluteTableIdentifier identifier;
@@ -61,10 +60,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
// segmentId -> list of index file
private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
- // segmentId -> SegmentProperties.
- private Map<String, SegmentProperties> segmentPropertiesMap = new HashMap<>();
-
- private Cache<TableBlockIndexUniqueIdentifier, DataMap> cache;
+ private Cache<TableBlockIndexUniqueIdentifier, AbstractCoarseGrainDataMap> cache;
@Override
public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
@@ -74,12 +70,12 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override
- public DataMapWriter createWriter(String segmentId) {
+ public AbstractDataMapWriter createWriter(String segmentId, String dataWriterPath) {
throw new UnsupportedOperationException("not implemented");
}
@Override
- public List<DataMap> getDataMaps(String segmentId) throws IOException {
+ public List<AbstractCoarseGrainDataMap> getDataMaps(String segmentId) throws IOException {
List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
getTableBlockIndexUniqueIdentifiers(segmentId);
return cache.getAll(tableBlockIndexUniqueIdentifiers);
@@ -140,17 +136,18 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
private ExtendedBlocklet getExtendedBlocklet(List<TableBlockIndexUniqueIdentifier> identifiers,
Blocklet blocklet) throws IOException {
- String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getPath());
+ String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getBlockId());
for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
if (identifier.getCarbonIndexFileName().equals(carbonIndexFileName)) {
DataMap dataMap = cache.get(identifier);
return ((BlockletDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId());
}
}
- throw new IOException("Blocklet with blockid " + blocklet.getPath() + " not found ");
+ throw new IOException("Blocklet with blockid " + blocklet.getBlockletId() + " not found ");
}
+
@Override
public List<DataMapDistributable> toDistributable(String segmentId) {
CarbonFile[] carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentId);
@@ -179,7 +176,6 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
@Override
public void clear(String segmentId) {
- segmentPropertiesMap.remove(segmentId);
List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segmentId);
if (blockIndexes != null) {
for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
@@ -200,7 +196,8 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override
- public List<DataMap> getDataMaps(DataMapDistributable distributable) throws IOException {
+ public List<AbstractCoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
if (mapDistributable.getFilePath().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
@@ -217,7 +214,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
indexFile));
}
}
- List<DataMap> dataMaps;
+ List<AbstractCoarseGrainDataMap> dataMaps;
try {
dataMaps = cache.getAll(identifiers);
} catch (IOException e) {
@@ -233,23 +230,20 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override public SegmentProperties getSegmentProperties(String segmentId) throws IOException {
- SegmentProperties segmentProperties = segmentPropertiesMap.get(segmentId);
- if (segmentProperties == null) {
- int[] columnCardinality;
- List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
- getTableBlockIndexUniqueIdentifiers(segmentId);
- DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
- List<DataFileFooter> indexInfo =
- fileFooterConverter.getIndexInfo(tableBlockIndexUniqueIdentifiers.get(0).getFilePath());
- for (DataFileFooter fileFooter : indexInfo) {
- List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
- if (segmentProperties == null) {
- columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
- segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
- }
- }
- segmentPropertiesMap.put(segmentId, segmentProperties);
+ List<AbstractCoarseGrainDataMap> dataMaps = getDataMaps(segmentId);
+ assert (dataMaps.size() > 0);
+ AbstractCoarseGrainDataMap coarseGrainDataMap = dataMaps.get(0);
+ assert (coarseGrainDataMap instanceof BlockletDataMap);
+ BlockletDataMap dataMap = (BlockletDataMap) coarseGrainDataMap;
+ return dataMap.getSegmentProperties();
+ }
+
+ @Override public List<Blocklet> getAllBlocklets(String segmentId) throws IOException {
+ List<Blocklet> blocklets = new ArrayList<>();
+ List<AbstractCoarseGrainDataMap> dataMaps = getDataMaps(segmentId);
+ for (AbstractCoarseGrainDataMap dataMap : dataMaps) {
+ blocklets.addAll(dataMap.prune(null, null));
}
- return segmentProperties;
+ return blocklets;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
index 5e0f4cf..0c83f91 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -29,7 +30,9 @@ import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* wrapper for blocklet data map data
@@ -44,6 +47,8 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
private BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache;
+ private BlockletSerializer blockletSerializer;
+
public BlockletDataRefNodeWrapper(List<TableBlockInfo> blockInfos, int index,
int[] dimensionLens) {
this.blockInfos = blockInfos;
@@ -55,6 +60,7 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
}
this.index = index;
this.dimensionLens = dimensionLens;
+ this.blockletSerializer = new BlockletSerializer();
}
@Override public DataRefNode getNextDataRefNode() {
@@ -130,14 +136,31 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
this.deleteDeltaDataCache = deleteDeltaDataCache;
}
- @Override public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
+ @Override
+ public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
return deleteDeltaDataCache;
}
- @Override public int numberOfPages() {
+ @Override
+ public int numberOfPages() {
return blockInfos.get(index).getDetailInfo().getPagesCount();
}
+ @Override
+ public BitSetGroup getIndexedData() {
+ String dataMapWriterPath = blockInfos.get(index).getDataMapWriterPath();
+ if (dataMapWriterPath != null) {
+ try {
+ FineGrainBlocklet blocklet = blockletSerializer.deserializeBlocklet(dataMapWriterPath);
+ return blocklet.getBitSetGroup(numberOfPages());
+ } catch (IOException e) {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
public int numberOfNodes() {
return blockInfos.size();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
index b8cffc6..2720700 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
@@ -22,6 +22,8 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -31,7 +33,10 @@ import org.apache.carbondata.core.util.CarbonUtil;
*/
public class IndexWrapper extends AbstractIndex {
+ private List<TableBlockInfo> blockInfos;
+
public IndexWrapper(List<TableBlockInfo> blockInfos) {
+ this.blockInfos = blockInfos;
DataFileFooter fileFooter = null;
try {
fileFooter = CarbonUtil.readMetadatFile(blockInfos.get(0));
@@ -46,4 +51,17 @@ public class IndexWrapper extends AbstractIndex {
@Override public void buildIndex(List<DataFileFooter> footerList) {
}
+
+ @Override public void clear() {
+ super.clear();
+ if (blockInfos != null) {
+ for (TableBlockInfo blockInfo : blockInfos) {
+ String dataMapWriterPath = blockInfo.getDataMapWriterPath();
+ if (dataMapWriterPath != null) {
+ CarbonFile file = FileFactory.getCarbonFile(dataMapWriterPath);
+ FileFactory.deleteAllCarbonFilesOfDir(file);
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
deleted file mode 100644
index 9d77010..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore.schema;
-
-/**
- * Types of filters of select query
- */
-public enum FilterType {
- EQUALTO, GREATER_THAN, LESS_THAN, GREATER_THAN_EQUAL, LESS_THAN_EQUAL, LIKE
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index f680579..886b12b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -313,6 +313,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ if (prvPageBitSet == null || prvPageBitSet.isEmpty()) {
+ return prvPageBitSet;
+ }
BitSet bitSet = new BitSet();
bitSet.or(prvPageBitSet);
byte[][] filterKeys = dimColumnExecuterInfo.getExcludeFilterKeys();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index fe1421c..0385c73 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -332,6 +332,9 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ if (prvPageBitSet == null || prvPageBitSet.isEmpty()) {
+ return prvPageBitSet;
+ }
BitSet bitSet = new BitSet(numberOfRows);
byte[][] filterKeys = dimColumnExecuterInfo.getFilterKeys();
int compareResult = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
index 60090d0..0ed9137 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
@@ -50,11 +50,6 @@ public class BlocksChunkHolder {
private BitSetGroup bitSetGroup;
- public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock) {
- dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
- measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock];
- }
-
public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock,
FileHolder fileReader) {
dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index abfc5f4..6e8076e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -153,6 +153,8 @@ public class FilterScanner extends AbstractBlockletScanner {
.get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
totalBlockletStatistic.getCount() + 1);
+ // set the indexed data if it has any during fgdatamap pruning.
+ blocksChunkHolder.setBitSetGroup(blocksChunkHolder.getDataBlock().getIndexedData());
// apply filter on actual data
BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder, useBitSetPipeLine);
// if indexes is empty then return with empty result
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 9d6acb6..cf90413 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -53,6 +54,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
@@ -97,6 +99,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.TBase;
import org.apache.thrift.TException;
@@ -129,6 +132,13 @@ public final class CarbonUtil {
private static final Configuration conf = new Configuration(true);
+ /**
+ * dfs.bytes-per-checksum
+ * HDFS checksum length, block size for a file should be exactly divisible
+ * by this value
+ */
+ private static final int HDFS_CHECKSUM_LENGTH = 512;
+
private CarbonUtil() {
}
@@ -2292,5 +2302,92 @@ public final class CarbonUtil {
return dataAndIndexSize;
}
+
+ /**
+ * This method will copy the given file to carbon store location
+ *
+ * @param localFilePath local file name with full path
+ * @throws CarbonDataWriterException
+ */
+ public static void copyCarbonDataFileToCarbonStorePath(String localFilePath,
+ String carbonDataDirectoryPath, long fileSizeInBytes)
+ throws CarbonDataWriterException {
+ long copyStartTime = System.currentTimeMillis();
+ LOGGER.info("Copying " + localFilePath + " --> " + carbonDataDirectoryPath);
+ try {
+ CarbonFile localCarbonFile =
+ FileFactory.getCarbonFile(localFilePath, FileFactory.getFileType(localFilePath));
+ String carbonFilePath = carbonDataDirectoryPath + localFilePath
+ .substring(localFilePath.lastIndexOf(File.separator));
+ copyLocalFileToCarbonStore(carbonFilePath, localFilePath,
+ CarbonCommonConstants.BYTEBUFFER_SIZE,
+ getMaxOfBlockAndFileSize(fileSizeInBytes, localCarbonFile.getSize()));
+ } catch (IOException e) {
+ throw new CarbonDataWriterException(
+ "Problem while copying file from local store to carbon store", e);
+ }
+ LOGGER.info(
+ "Total copy time (ms) to copy file " + localFilePath + " is " + (System.currentTimeMillis()
+ - copyStartTime));
+ }
+
+ /**
+ * This method will read the local carbon data file and write to carbon data file in HDFS
+ *
+ * @param carbonStoreFilePath
+ * @param localFilePath
+ * @param bufferSize
+ * @param blockSize
+ * @throws IOException
+ */
+ private static void copyLocalFileToCarbonStore(String carbonStoreFilePath, String localFilePath,
+ int bufferSize, long blockSize) throws IOException {
+ DataOutputStream dataOutputStream = null;
+ DataInputStream dataInputStream = null;
+ try {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize
+ + " (bytes");
+ }
+ dataOutputStream = FileFactory
+ .getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
+ bufferSize, blockSize);
+ dataInputStream = FileFactory
+ .getDataInputStream(localFilePath, FileFactory.getFileType(localFilePath), bufferSize);
+ IOUtils.copyBytes(dataInputStream, dataOutputStream, bufferSize);
+ } finally {
+ CarbonUtil.closeStream(dataInputStream);
+ CarbonUtil.closeStream(dataOutputStream);
+ }
+ }
+
+ /**
+ * This method will return max of block size and file size
+ *
+ * @param blockSize
+ * @param fileSize
+ * @return
+ */
+ private static long getMaxOfBlockAndFileSize(long blockSize, long fileSize) {
+ long maxSize = blockSize;
+ if (fileSize > blockSize) {
+ maxSize = fileSize;
+ }
+ // block size should be exactly divisible by 512 which is maintained by HDFS as bytes
+ // per checksum, dfs.bytes-per-checksum=512 must divide block size
+ long remainder = maxSize % HDFS_CHECKSUM_LENGTH;
+ if (remainder > 0) {
+ maxSize = maxSize + HDFS_CHECKSUM_LENGTH - remainder;
+ }
+ // convert to make block size more readable.
+ String readableBlockSize = ByteUtil.convertByteToReadable(blockSize);
+ String readableFileSize = ByteUtil.convertByteToReadable(fileSize);
+ String readableMaxSize = ByteUtil.convertByteToReadable(maxSize);
+ LOGGER.info(
+ "The configured block size is " + readableBlockSize + ", the actual carbon file size is "
+ + readableFileSize + ", choose the max value " + readableMaxSize
+ + " as the block size on HDFS");
+ return maxSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
index 2ad6327..605796f 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
@@ -28,7 +28,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
@@ -47,7 +48,7 @@ import com.google.gson.Gson;
/**
* Datamap implementation for min max blocklet.
*/
-public class MinMaxDataMap implements DataMap {
+public class MinMaxDataMap extends AbstractCoarseGrainDataMap {
private static final LogService LOGGER =
LogServiceFactory.getLogService(MinMaxDataMap.class.getName());
@@ -58,8 +59,8 @@ public class MinMaxDataMap implements DataMap {
private MinMaxIndexBlockDetails[] readMinMaxDataMap;
- @Override public void init(String filePath) throws MemoryException, IOException {
- this.filePath = filePath;
+ @Override public void init(DataMapModel model) throws MemoryException, IOException {
+ this.filePath = model.getFilePath();
CarbonFile[] listFiles = getCarbonMinMaxIndexFiles(filePath, "0");
for (int i = 0; i < listFiles.length; i++) {
readMinMaxDataMap = readJson(listFiles[i].getPath());
@@ -90,8 +91,7 @@ public class MinMaxDataMap implements DataMap {
return null;
}
dataInputStream = fileOperation.openForRead();
- inStream = new InputStreamReader(dataInputStream,
- CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
+ inStream = new InputStreamReader(dataInputStream, "UTF-8");
buffReader = new BufferedReader(inStream);
readMinMax = gsonObjectToRead.fromJson(buffReader, MinMaxIndexBlockDetails[].class);
} catch (IOException e) {
@@ -115,8 +115,7 @@ public class MinMaxDataMap implements DataMap {
if (filterExp == null) {
for (int i = 0; i < readMinMaxDataMap.length; i++) {
- blocklets.add(new Blocklet(readMinMaxDataMap[i].getFilePath(),
- String.valueOf(readMinMaxDataMap[i].getBlockletId())));
+ blocklets.add(new Blocklet(filePath, String.valueOf(readMinMaxDataMap[i].getBlockletId())));
}
} else {
FilterExecuter filterExecuter =
@@ -126,7 +125,7 @@ public class MinMaxDataMap implements DataMap {
BitSet bitSet = filterExecuter.isScanRequired(readMinMaxDataMap[startIndex].getMaxValues(),
readMinMaxDataMap[startIndex].getMinValues());
if (!bitSet.isEmpty()) {
- blocklets.add(new Blocklet(readMinMaxDataMap[startIndex].getFilePath(),
+ blocklets.add(new Blocklet(filePath,
String.valueOf(readMinMaxDataMap[startIndex].getBlockletId())));
}
startIndex++;
@@ -135,8 +134,7 @@ public class MinMaxDataMap implements DataMap {
return blocklets;
}
- @Override
- public void clear() {
+ @Override public void clear() {
readMinMaxDataMap = null;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
index b196d0d..5203cb3 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -25,49 +25,51 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
-import org.apache.carbondata.core.events.ChangeEvent;
-import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.events.Event;
/**
* Min Max DataMap Factory
*/
-public class MinMaxDataMapFactory implements DataMapFactory {
+public class MinMaxDataMapFactory extends AbstractCoarseGrainDataMapFactory {
private AbsoluteTableIdentifier identifier;
- @Override
- public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ @Override public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
this.identifier = identifier;
}
/**
* createWriter will return the MinMaxDataWriter.
+ *
* @param segmentId
* @return
*/
- @Override
- public DataMapWriter createWriter(String segmentId) {
- return new MinMaxDataWriter();
+ @Override public AbstractDataMapWriter createWriter(String segmentId, String dataWritePath) {
+ return new MinMaxDataWriter(identifier, segmentId, dataWritePath);
}
/**
* getDataMaps Factory method Initializes the Min Max Data Map and returns.
+ *
* @param segmentId
* @return
* @throws IOException
*/
- @Override public List<DataMap> getDataMaps(String segmentId) throws IOException {
- List<DataMap> dataMapList = new ArrayList<>();
+ @Override public List<AbstractCoarseGrainDataMap> getDataMaps(String segmentId)
+ throws IOException {
+ List<AbstractCoarseGrainDataMap> dataMapList = new ArrayList<>();
// Form a dataMap of Type MinMaxDataMap.
MinMaxDataMap dataMap = new MinMaxDataMap();
try {
- dataMap.init(identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + File.separator);
+ dataMap.init(new DataMapModel(
+ identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + File.separator));
} catch (MemoryException ex) {
}
@@ -76,7 +78,6 @@ public class MinMaxDataMapFactory implements DataMapFactory {
}
/**
- *
* @param segmentId
* @return
*/
@@ -86,6 +87,7 @@ public class MinMaxDataMapFactory implements DataMapFactory {
/**
* Clear the DataMap.
+ *
* @param segmentId
*/
@Override public void clear(String segmentId) {
@@ -94,21 +96,20 @@ public class MinMaxDataMapFactory implements DataMapFactory {
/**
* Clearing the data map.
*/
- @Override
- public void clear() {
+ @Override public void clear() {
}
- @Override public DataMap getDataMap(DataMapDistributable distributable) {
+ @Override public List<AbstractCoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
return null;
}
- @Override
- public void fireEvent(ChangeEvent event) {
+ @Override public void fireEvent(Event event) {
}
- @Override
- public DataMapMeta getMeta() {
- return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")), FilterType.EQUALTO);
+ @Override public DataMapMeta getMeta() {
+ return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")),
+ new ArrayList<ExpressionType>());
}
}
\ No newline at end of file
[17/28] carbondata git commit: [CARBONDATA-1752] There are some
scalastyle error should be optimized in CarbonData
Posted by ja...@apache.org.
[CARBONDATA-1752] There are some scalastyle error should be optimized in CarbonData
There are some scalastyle error should be optimized in CarbonData
This closes #1518
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1d2af629
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1d2af629
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1d2af629
Branch: refs/heads/fgdatamap
Commit: 1d2af629367b906f5424819f94e208aa9f30db4d
Parents: dfc7442
Author: xubo245 <60...@qq.com>
Authored: Fri Nov 17 11:25:14 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Nov 17 17:23:50 2017 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/api/CarbonStore.scala | 2 +-
.../carbondata/spark/util/CommonUtil.scala | 5 ++--
.../spark/util/DataTypeConverterUtil.scala | 2 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +-
.../command/carbonTableSchemaCommon.scala | 25 +++++++++++---------
5 files changed, 19 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d2af629/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index 3d93a65..a2c9c6d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -29,7 +29,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d2af629/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index a3572ed..6c0e802 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -27,10 +27,9 @@ import scala.collection.mutable.Map
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
import org.apache.spark.SparkContext
-import org.apache.spark.sql.{Row, RowFactory, SQLContext}
+import org.apache.spark.sql.{Row, RowFactory}
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
import org.apache.spark.sql.execution.command.{ColumnProperty, Field, PartitionerField}
import org.apache.spark.sql.types.{MetadataBuilder, StringType}
@@ -41,7 +40,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.filesystem.CarbonFile
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.memory.{UnsafeMemoryManager, UnsafeSortMemoryManager}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier}
import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
import org.apache.carbondata.core.metadata.schema.PartitionInfo
import org.apache.carbondata.core.metadata.schema.partition.PartitionType
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d2af629/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 6cf7298..38657ac 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -17,7 +17,7 @@
package org.apache.carbondata.spark.util
-import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes, DecimalType}
+import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
import org.apache.carbondata.format.{DataType => ThriftDataType}
object DataTypeConverterUtil {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d2af629/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index e83d96a..0e77e8c 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -39,7 +39,7 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.PartitionInfo
import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, DataTypeUtil}
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.{CommonUtil, DataTypeConverterUtil}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d2af629/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 2f6b277..9c927aa 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -103,7 +103,8 @@ case class DataLoadTableFileMapping(table: String, loadPath: String)
case class ExecutionErrors(var failureCauses: FailureCauses, var errorMsg: String )
-case class CarbonMergerMapping(hdfsStoreLocation: String,
+case class CarbonMergerMapping(
+ hdfsStoreLocation: String,
metadataFilePath: String,
var mergedLoadName: String,
databaseName: String,
@@ -118,16 +119,18 @@ case class CarbonMergerMapping(hdfsStoreLocation: String,
case class NodeInfo(TaskId: String, noOfBlocks: Int)
-case class AlterTableModel(dbName: Option[String],
- tableName: String,
- segmentUpdateStatusManager: Option[SegmentUpdateStatusManager],
- compactionType: String,
- factTimeStamp: Option[Long],
- var alterSql: String)
-
-case class UpdateTableModel(isUpdate: Boolean,
- updatedTimeStamp: Long,
- var executorErrors: ExecutionErrors)
+case class AlterTableModel(
+ dbName: Option[String],
+ tableName: String,
+ segmentUpdateStatusManager: Option[SegmentUpdateStatusManager],
+ compactionType: String,
+ factTimeStamp: Option[Long],
+ var alterSql: String)
+
+case class UpdateTableModel(
+ isUpdate: Boolean,
+ updatedTimeStamp: Long,
+ var executorErrors: ExecutionErrors)
case class CompactionModel(compactionSize: Long,
compactionType: CompactionType,
[03/28] carbondata git commit: handled review comments
Posted by ja...@apache.org.
handled review comments
add column comment during carbon create table and when table is described if comment is not mentioned, default will be null
added test case when sort column is boolean column
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9c9521b6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9c9521b6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9c9521b6
Branch: refs/heads/fgdatamap
Commit: 9c9521b683fe19458d243c813dc622d30e06074d
Parents: 17892b1
Author: akashrn5 <ak...@gmail.com>
Authored: Tue Oct 24 15:56:53 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Thu Nov 16 20:41:00 2017 +0530
----------------------------------------------------------------------
.../TestCreateTableWithColumnComment.scala | 54 ++++++++++++++++++++
.../CarbonDescribeFormattedCommand.scala | 19 ++++---
.../sql/parser/CarbonSpark2SqlParser.scala | 14 +++--
.../BooleanDataTypesInsertTest.scala | 40 +++++++++++++++
4 files changed, 115 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9c9521b6/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithColumnComment.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithColumnComment.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithColumnComment.scala
new file mode 100644
index 0000000..c291a6f
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableWithColumnComment.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.createTable
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test functionality of create table with column comment
+ */
+class TestCreateTableWithColumnComment extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ sql("use default")
+ sql("drop table if exists columnComment")
+ sql("drop table if exists defaultComment")
+ }
+
+ test("test create table with column comment") {
+ sql(
+ "create table columnComment(id int, name string comment \"This column is called name\") " +
+ "stored by 'carbondata'")
+ checkExistence(sql("describe formatted columnComment"), true, "This column is called name")
+ }
+
+ test("test create table with default column comment value") {
+ sql(
+ "create table defaultComment(id int, name string) " +
+ "stored by 'carbondata'")
+ checkExistence(sql("describe formatted defaultComment"), true, "null")
+ }
+
+ override def afterAll {
+ sql("use default")
+ sql("drop table if exists columnComment")
+ sql("drop table if exists defaultComment")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9c9521b6/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
index 519fbea..7dcad9a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
@@ -65,6 +65,7 @@ private[sql] case class CarbonDescribeFormattedCommand(
val dims = relation.metaData.dims.map(x => x.toLowerCase)
var results: Seq[(String, String, String)] = child.schema.fields.map { field =>
val fieldName = field.name.toLowerCase
+ val colComment = field.getComment().getOrElse("null")
val comment = if (dims.contains(fieldName)) {
val dimension = relation.metaData.carbonTable.getDimensionByName(
relation.tableMeta.carbonTableIdentifier.getTableName, fieldName)
@@ -76,20 +77,21 @@ private[sql] case class CarbonDescribeFormattedCommand(
if (dimension.hasEncoding(Encoding.DICTIONARY) &&
!dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
"DICTIONARY, KEY COLUMN" + (if (dimension.hasEncoding(Encoding.INVERTED_INDEX)) {
- ""
+ "".concat(",").concat(colComment)
} else {
- ",NOINVERTEDINDEX"
+ ",NOINVERTEDINDEX".concat(",").concat(colComment)
})
} else {
"KEY COLUMN" + (if (dimension.hasEncoding(Encoding.INVERTED_INDEX)) {
- ""
+ "".concat(",").concat(colComment)
} else {
- ",NOINVERTEDINDEX"
+ ",NOINVERTEDINDEX".concat(",").concat(colComment)
})
}
} else {
- "MEASURE"
+ "MEASURE".concat(",").concat(colComment)
}
+
(field.name, field.dataType.simpleString, comment)
}
val colPropStr = if (colProps.toString().trim().length() > 0) {
@@ -130,8 +132,11 @@ private[sql] case class CarbonDescribeFormattedCommand(
Seq(("Partition Columns: ", carbonTable.getPartitionInfo(carbonTable.getFactTableName)
.getColumnSchemaList.asScala.map(_.getColumnName).mkString(","), ""))
}
- results.map { case (name, dataType, comment) =>
- Row(f"$name%-36s", f"$dataType%-80s", f"$comment%-72s")
+ results.map {
+ case (name, dataType, null) =>
+ Row(f"$name%-36s", f"$dataType%-80s", null)
+ case (name, dataType, comment) =>
+ Row(f"$name%-36s", f"$dataType%-80s", f"$comment%-72s")
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9c9521b6/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index 6df5a04..7a637f7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -465,12 +465,16 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
def getFields(schema: Seq[StructField]): Seq[Field] = {
schema.map { col =>
- val x = if (col.dataType.catalogString == "float") {
- '`' + col.name + '`' + " double"
- }
- else {
- '`' + col.name + '`' + ' ' + col.dataType.catalogString
+ var columnComment: String = ""
+ if (col.getComment().isDefined) {
+ columnComment = " comment \"" + col.getComment().get + "\""
}
+ val x =
+ if (col.dataType.catalogString == "float") {
+ '`' + col.name + '`' + " double" + columnComment
+ } else {
+ '`' + col.name + '`' + ' ' + col.dataType.catalogString + columnComment
+ }
val f: Field = anyFieldDef(new lexical.Scanner(x.toLowerCase))
match {
case Success(field, _) => field.asInstanceOf[Field]
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9c9521b6/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
index 2f06900..c6a6708 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
@@ -22,6 +22,9 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
override def beforeEach(): Unit = {
@@ -945,4 +948,41 @@ class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with
)
}
+ test("Inserting table with bad records, and SORT_COLUMNS is boolean column") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
+ sql("DROP TABLE IF EXISTS carbon_table")
+ sql(
+ s"""
+ | CREATE TABLE if not exists carbon_table(
+ | cc BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('SORT_COLUMNS'='cc')
+ """.stripMargin)
+ sql("insert into carbon_table values(true)")
+ sql("insert into carbon_table values(True)")
+ sql("insert into carbon_table values(TRUE)")
+ sql("insert into carbon_table values('true')")
+ sql("insert into carbon_table values(False)")
+ sql("insert into carbon_table values(false)")
+ sql("insert into carbon_table values(FALSE)")
+ sql("insert into carbon_table values('false')")
+ sql("insert into carbon_table values('tr')")
+ sql("insert into carbon_table values(null)")
+ sql("insert into carbon_table values('truEe')")
+ sql("insert into carbon_table values('falSee')")
+ sql("insert into carbon_table values('t')")
+ sql("insert into carbon_table values('f')")
+ checkAnswer(
+ sql("select * from carbon_table"),
+ Seq(
+ Row(true), Row(true), Row(true), Row(true),
+ Row(false), Row(false), Row(false), Row(false),
+ Row(null), Row(null), Row(null), Row(null), Row(null), Row(null)))
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
+ CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)
+ }
+
}
[11/28] carbondata git commit: [CARBONDATA-1739] Clean up store path
interface
Posted by ja...@apache.org.
[CARBONDATA-1739] Clean up store path interface
This closes #1509
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5fc7f06f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5fc7f06f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5fc7f06f
Branch: refs/heads/fgdatamap
Commit: 5fc7f06f23e944719b2735b97176d68fe209ad75
Parents: b6777fc
Author: Jacky Li <ja...@qq.com>
Authored: Thu Nov 16 19:41:19 2017 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Fri Nov 17 14:46:19 2017 +0800
----------------------------------------------------------------------
.../dictionary/ManageDictionaryAndBTree.java | 2 +-
.../core/metadata/CarbonMetadata.java | 2 +-
.../core/metadata/schema/table/CarbonTable.java | 4 +-
.../core/mutate/CarbonUpdateUtil.java | 8 +-
.../carbondata/core/scan/model/QueryModel.java | 4 +-
.../carbondata/core/util/CarbonProperties.java | 7 +
.../core/metadata/CarbonMetadataTest.java | 9 +-
.../metadata/schema/table/CarbonTableTest.java | 3 +-
.../table/CarbonTableWithComplexTypesTest.java | 2 +-
.../carbondata/examples/StreamExample.scala | 4 +-
.../carbondata/hadoop/CarbonInputFormat.java | 2 +-
.../hadoop/api/CarbonTableInputFormat.java | 4 +-
.../streaming/CarbonStreamRecordReader.java | 10 +-
.../streaming/CarbonStreamRecordWriter.java | 4 +-
.../hadoop/util/CarbonInputFormatUtil.java | 6 +-
.../hadoop/test/util/StoreCreator.java | 4 +-
.../presto/impl/CarbonTableReader.java | 2 +-
.../presto/util/CarbonDataStoreCreator.scala | 4 +-
.../TestPreAggregateTableSelection.scala | 2 +-
.../partition/TestDDLForPartitionTable.scala | 6 +-
...ForPartitionTableWithDefaultProperties.scala | 8 +-
.../carbondata/spark/load/ValidateUtil.scala | 4 +-
.../spark/rdd/AlterTableLoadPartitionRDD.scala | 2 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala | 2 +-
.../carbondata/spark/rdd/PartitionDropper.scala | 2 +-
.../spark/rdd/PartitionSplitter.scala | 2 +-
.../carbondata/spark/util/CommonUtil.scala | 32 +----
.../carbondata/spark/util/DataLoadingUtil.scala | 8 +-
.../spark/util/GlobalDictionaryUtil.scala | 12 +-
.../command/carbonTableSchemaCommon.scala | 4 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 12 +-
.../carbondata/spark/util/CarbonSparkUtil.scala | 18 ++-
.../spark/sql/CarbonDataFrameWriter.scala | 3 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 4 +-
.../spark/sql/CarbonDictionaryDecoder.scala | 16 +--
.../scala/org/apache/spark/sql/CarbonEnv.scala | 36 ++++-
.../scala/org/apache/spark/sql/CarbonScan.scala | 6 +-
.../org/apache/spark/sql/CarbonSource.scala | 20 +--
.../command/CarbonCreateTableCommand.scala | 4 +-
.../CarbonDescribeFormattedCommand.scala | 20 +--
.../command/CarbonDropTableCommand.scala | 9 +-
.../datamap/CarbonDataMapShowCommand.scala | 4 +-
.../datamap/CarbonDropDataMapCommand.scala | 31 +++--
.../AlterTableCompactionCommand.scala | 12 +-
.../management/CarbonShowLoadsCommand.scala | 4 +-
.../command/management/CleanFilesCommand.scala | 10 +-
.../management/DeleteLoadByIdCommand.scala | 4 +-
.../DeleteLoadByLoadDateCommand.scala | 4 +-
.../management/LoadTableByInsertCommand.scala | 2 +-
.../command/management/LoadTableCommand.scala | 62 ++++-----
.../command/mutation/DeleteExecution.scala | 13 +-
.../command/mutation/HorizontalCompaction.scala | 8 +-
.../command/mutation/IUDCommonUtil.scala | 2 +-
.../mutation/ProjectForDeleteCommand.scala | 7 +-
.../mutation/ProjectForUpdateCommand.scala | 11 +-
.../AlterTableDropCarbonPartitionCommand.scala | 19 +--
.../AlterTableSplitCarbonPartitionCommand.scala | 19 +--
.../partition/ShowCarbonPartitionsCommand.scala | 7 +-
.../CreatePreAggregateTableCommand.scala | 7 +-
.../preaaggregate/PreAggregateListeners.scala | 6 +-
.../preaaggregate/PreAggregateUtil.scala | 37 +++---
.../CarbonAlterTableAddColumnCommand.scala | 4 +-
.../CarbonAlterTableDataTypeChangeCommand.scala | 4 +-
.../CarbonAlterTableDropColumnCommand.scala | 4 +-
.../schema/CarbonAlterTableRenameCommand.scala | 7 +-
.../strategy/CarbonLateDecodeStrategy.scala | 4 +-
.../sql/execution/strategy/DDLStrategy.scala | 11 +-
.../strategy/StreamingTableStrategy.scala | 3 +-
.../spark/sql/hive/CarbonFileMetastore.scala | 61 ++++-----
.../spark/sql/hive/CarbonHiveMetaStore.scala | 13 +-
.../apache/spark/sql/hive/CarbonMetaStore.scala | 4 +-
.../sql/hive/CarbonPreAggregateRules.scala | 2 +-
.../apache/spark/sql/hive/CarbonRelation.scala | 26 ++--
.../spark/sql/hive/CarbonSessionState.scala | 13 +-
.../execution/command/CarbonHiveCommands.scala | 4 +-
.../org/apache/spark/util/AlterTableUtil.scala | 36 ++---
.../org/apache/spark/util/CleanFiles.scala | 5 +-
.../apache/spark/util/DeleteSegmentByDate.scala | 5 +-
.../apache/spark/util/DeleteSegmentById.scala | 4 +-
.../partition/TestAlterPartitionTable.scala | 32 ++---
.../spark/util/AllDictionaryTestCase.scala | 16 +--
.../spark/util/DictionaryTestCaseUtil.scala | 6 +-
.../util/ExternalColumnDictionaryTestCase.scala | 16 +--
.../loading/DataLoadProcessBuilder.java | 6 +-
.../merger/CarbonCompactionExecutor.java | 4 +-
.../processing/merger/CarbonCompactionUtil.java | 4 +-
.../processing/merger/CarbonDataMergerUtil.java | 8 +-
.../carbondata/processing/merger/TableMeta.java | 42 ------
.../spliter/AbstractCarbonQueryExecutor.java | 4 +-
.../partition/spliter/RowResultProcessor.java | 2 +-
.../store/CarbonFactDataHandlerColumnar.java | 130 -------------------
.../processing/store/file/FileData.java | 52 --------
.../processing/store/file/FileManager.java | 59 ---------
.../store/file/IFileManagerComposite.java | 57 --------
.../store/writer/AbstractFactDataWriter.java | 4 -
.../store/writer/CarbonDataWriterVo.java | 65 ----------
.../util/CarbonDataProcessorUtil.java | 2 +-
.../processing/util/CarbonLoaderUtil.java | 5 +
.../carbondata/processing/StoreCreator.java | 4 +-
.../streaming/segment/StreamSegment.java | 16 +--
.../streaming/StreamSinkFactory.scala | 2 +-
.../CarbonStreamingQueryListener.scala | 6 +-
102 files changed, 423 insertions(+), 911 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
index a6c89e0..f8d2495 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
@@ -112,7 +112,7 @@ public class ManageDictionaryAndBTree {
}
// clear dictionary cache from LRU cache
List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
+ carbonTable.getDimensionByTableName(carbonTable.getTableName());
for (CarbonDimension dimension : dimensions) {
removeDictionaryColumnFromCache(carbonTable.getAbsoluteTableIdentifier(),
dimension.getColumnId());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
index 75fe78b..2face7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
@@ -120,7 +120,7 @@ public final class CarbonMetadata {
public CarbonDimension getCarbonDimensionBasedOnColIdentifier(CarbonTable carbonTable,
String columnIdentifier) {
List<CarbonDimension> listOfCarbonDims =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
+ carbonTable.getDimensionByTableName(carbonTable.getTableName());
for (CarbonDimension dimension : listOfCarbonDims) {
if (dimension.getColumnId().equals(columnIdentifier)) {
return dimension;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index f76ddc9..ac580cd 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -347,7 +347,7 @@ public class CarbonTable implements Serializable {
/**
* @return the tabelName
*/
- public String getFactTableName() {
+ public String getTableName() {
return absoluteTableIdentifier.getCarbonTableIdentifier().getTableName();
}
@@ -569,7 +569,7 @@ public class CarbonTable implements Serializable {
}
public boolean isPartitionTable() {
- return null != tablePartitionMap.get(getFactTableName());
+ return null != tablePartitionMap.get(getTableName());
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 29cf62a..0b531dc 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -208,7 +208,7 @@ public class CarbonUpdateUtil {
lockStatus = carbonLock.lockWithRetries();
if (lockStatus) {
LOGGER.info(
- "Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName()
+ "Acquired lock for table" + table.getDatabaseName() + "." + table.getTableName()
+ " for table status updation");
LoadMetadataDetails[] listOfLoadFolderDetailsArray =
@@ -257,18 +257,18 @@ public class CarbonUpdateUtil {
status = true;
} else {
LOGGER.error("Not able to acquire the lock for Table status updation for table " + table
- .getDatabaseName() + "." + table.getFactTableName());
+ .getDatabaseName() + "." + table.getTableName());
}
} finally {
if (lockStatus) {
if (carbonLock.unlock()) {
LOGGER.info(
"Table unlocked successfully after table status updation" + table.getDatabaseName()
- + "." + table.getFactTableName());
+ + "." + table.getTableName());
} else {
LOGGER.error(
"Unable to unlock Table lock for table" + table.getDatabaseName() + "." + table
- .getFactTableName() + " during table status updation");
+ .getTableName() + " during table status updation");
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 66dfa61..67b8681 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -122,7 +122,7 @@ public class QueryModel implements Serializable {
public static QueryModel createModel(AbsoluteTableIdentifier absoluteTableIdentifier,
CarbonQueryPlan queryPlan, CarbonTable carbonTable, DataTypeConverter converter) {
QueryModel queryModel = new QueryModel();
- String factTableName = carbonTable.getFactTableName();
+ String factTableName = carbonTable.getTableName();
queryModel.setAbsoluteTableIdentifier(absoluteTableIdentifier);
fillQueryModel(queryPlan, carbonTable, queryModel, factTableName);
@@ -141,7 +141,7 @@ public class QueryModel implements Serializable {
if (null != queryPlan.getFilterExpression()) {
boolean[] isFilterDimensions = new boolean[carbonTable.getDimensionOrdinalMax()];
boolean[] isFilterMeasures =
- new boolean[carbonTable.getNumberOfMeasures(carbonTable.getFactTableName())];
+ new boolean[carbonTable.getNumberOfMeasures(carbonTable.getTableName())];
processFilterExpression(queryPlan.getFilterExpression(),
carbonTable.getDimensionByTableName(factTableName),
carbonTable.getMeasureByTableName(factTableName), isFilterDimensions, isFilterMeasures);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 678a6f7..436950b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -482,6 +482,13 @@ public final class CarbonProperties {
}
/**
+ * Return the store path
+ */
+ public static String getStorePath() {
+ return getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION);
+ }
+
+ /**
* This method will be used to get the properties value
*
* @param key
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
index 0de160a..5361fb0 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
@@ -20,7 +20,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
-import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -94,7 +93,7 @@ public class CarbonMetadataTest {
@Test public void testGetCarbonTableReturingProperTableWithProperFactTableName() {
String expectedResult = "carbonTestTable";
- assertEquals(expectedResult, carbonMetadata.getCarbonTable(tableUniqueName).getFactTableName());
+ assertEquals(expectedResult, carbonMetadata.getCarbonTable(tableUniqueName).getTableName());
}
@Test public void testGetCarbonTableReturingProperTableWithProperTableUniqueName() {
@@ -171,7 +170,7 @@ public class CarbonMetadataTest {
carbonDimensions.add(new CarbonDimension(colSchema1, 1, 1, 2, 1));
carbonDimensions.add(new CarbonDimension(colSchema2, 2, 2, 2, 2));
new MockUp<CarbonTable>() {
- @Mock public String getFactTableName() {
+ @Mock public String getTableName() {
return "carbonTestTable";
}
@@ -200,7 +199,7 @@ public class CarbonMetadataTest {
colSchema2.setColumnUniqueId("2");
carbonChildDimensions.add(new CarbonDimension(colSchema3, 1, 1, 2, 1));
new MockUp<CarbonTable>() {
- @Mock public String getFactTableName() {
+ @Mock public String getTableName() {
return "carbonTestTable";
}
@@ -242,7 +241,7 @@ public class CarbonMetadataTest {
carbonChildDimensions.add(new CarbonDimension(colSchema2, 1, 1, 2, 1));
new MockUp<CarbonTable>() {
- @Mock public String getFactTableName() {
+ @Mock public String getTableName() {
return "carbonTestTable";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
index 8b66233..a47b7fd 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
@@ -20,7 +20,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
-import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -57,7 +56,7 @@ public class CarbonTableTest extends TestCase {
}
@Test public void testFactTableNameReturnsProperFactTableName() {
- assertEquals("carbonTestTable", carbonTable.getFactTableName());
+ assertEquals("carbonTestTable", carbonTable.getTableName());
}
@Test public void testTableUniqueNameIsProper() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
index e9caf4a..84312cd 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
@@ -55,7 +55,7 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
}
@Test public void testFactTableNameReturnsProperFactTableName() {
- assertEquals("carbonTestTable", carbonTable.getFactTableName());
+ assertEquals("carbonTestTable", carbonTable.getTableName());
}
@Test public void testTableUniqueNameIsProper() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamExample.scala
index 4b59aad..43d545d 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamExample.scala
@@ -89,9 +89,7 @@ object StreamExample {
| """.stripMargin)
}
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.
- lookupRelation(Some("default"), streamTableName)(spark).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(Some("default"), streamTableName)(spark)
val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
// batch load
val path = s"$rootPath/examples/spark2/src/main/resources/streamSample.csv"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 0aa2974..88d8341 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -364,7 +364,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
TableProvider tableProvider = new SingleTableProvider(carbonTable);
CarbonInputFormatUtil.processFilterExpression(filter, carbonTable, null, null);
BitSet matchedPartitions = null;
- PartitionInfo partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName());
+ PartitionInfo partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName());
if (partitionInfo != null) {
// prune partitions for filter query on partition table
matchedPartitions = setMatchedPartitions(null, carbonTable, filter, partitionInfo);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 6e840e2..552455a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -393,7 +393,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
Expression filter = getFilterPredicates(job.getConfiguration());
TableProvider tableProvider = new SingleTableProvider(carbonTable);
// this will be null in case of corrupt schema file.
- PartitionInfo partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName());
+ PartitionInfo partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName());
CarbonInputFormatUtil.processFilterExpression(filter, carbonTable, null, null);
// prune partitions for filter query on partition table
@@ -787,7 +787,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
Expression filter = getFilterPredicates(configuration);
boolean[] isFilterDimensions = new boolean[carbonTable.getDimensionOrdinalMax()];
boolean[] isFilterMeasures =
- new boolean[carbonTable.getNumberOfMeasures(carbonTable.getFactTableName())];
+ new boolean[carbonTable.getNumberOfMeasures(carbonTable.getTableName())];
CarbonInputFormatUtil.processFilterExpression(filter, carbonTable, isFilterDimensions,
isFilterMeasures);
queryModel.setIsFilterDimensions(isFilterDimensions);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
index a22461d..bdd7c28 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
@@ -153,13 +153,13 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
}
carbonTable = model.getTable();
List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
+ carbonTable.getDimensionByTableName(carbonTable.getTableName());
dimensionCount = dimensions.size();
List<CarbonMeasure> measures =
- carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
+ carbonTable.getMeasureByTableName(carbonTable.getTableName());
measureCount = measures.size();
List<CarbonColumn> carbonColumnList =
- carbonTable.getStreamStorageOrderColumn(carbonTable.getFactTableName());
+ carbonTable.getStreamStorageOrderColumn(carbonTable.getTableName());
storageColumns = carbonColumnList.toArray(new CarbonColumn[carbonColumnList.size()]);
isNoDictColumn = CarbonDataProcessorUtil.getNoDictionaryMapping(storageColumns);
directDictionaryGenerators = new DirectDictionaryGenerator[storageColumns.length];
@@ -224,8 +224,8 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
private void initializeFilter() {
List<ColumnSchema> wrapperColumnSchemaList = CarbonUtil
- .getColumnSchemaList(carbonTable.getDimensionByTableName(carbonTable.getFactTableName()),
- carbonTable.getMeasureByTableName(carbonTable.getFactTableName()));
+ .getColumnSchemaList(carbonTable.getDimensionByTableName(carbonTable.getTableName()),
+ carbonTable.getMeasureByTableName(carbonTable.getTableName()));
int[] dimLensWithComplex = new int[wrapperColumnSchemaList.size()];
for (int i = 0; i < dimLensWithComplex.length; i++) {
dimLensWithComplex[i] = Integer.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
index 7df87e3..fdd0504 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
@@ -251,8 +251,8 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
private void writeFileHeader() throws IOException {
List<ColumnSchema> wrapperColumnSchemaList = CarbonUtil
- .getColumnSchemaList(carbonTable.getDimensionByTableName(carbonTable.getFactTableName()),
- carbonTable.getMeasureByTableName(carbonTable.getFactTableName()));
+ .getColumnSchemaList(carbonTable.getDimensionByTableName(carbonTable.getTableName()),
+ carbonTable.getMeasureByTableName(carbonTable.getTableName()));
int[] dimLensWithComplex = new int[wrapperColumnSchemaList.size()];
for (int i = 0; i < dimLensWithComplex.length; i++) {
dimLensWithComplex[i] = Integer.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 630828a..3afad94 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -52,7 +52,7 @@ public class CarbonInputFormatUtil {
if (columnString != null) {
columns = columnString.split(",");
}
- String factTableName = carbonTable.getFactTableName();
+ String factTableName = carbonTable.getTableName();
CarbonQueryPlan plan = new CarbonQueryPlan(carbonTable.getDatabaseName(), factTableName);
// fill dimensions
// If columns are null, set all dimensions and measures
@@ -120,9 +120,9 @@ public class CarbonInputFormatUtil {
public static void processFilterExpression(Expression filterExpression, CarbonTable carbonTable,
boolean[] isFilterDimensions, boolean[] isFilterMeasures) {
List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
+ carbonTable.getDimensionByTableName(carbonTable.getTableName());
List<CarbonMeasure> measures =
- carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
+ carbonTable.getMeasureByTableName(carbonTable.getTableName());
QueryModel.processFilterExpression(filterExpression, dimensions, measures,
isFilterDimensions, isFilterMeasures);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index b4145ef..c45f910 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -309,9 +309,9 @@ public class StoreCreator {
String header = reader.readLine();
String[] split = header.split(",");
List<CarbonColumn> allCols = new ArrayList<CarbonColumn>();
- List<CarbonDimension> dims = table.getDimensionByTableName(table.getFactTableName());
+ List<CarbonDimension> dims = table.getDimensionByTableName(table.getTableName());
allCols.addAll(dims);
- List<CarbonMeasure> msrs = table.getMeasureByTableName(table.getFactTableName());
+ List<CarbonMeasure> msrs = table.getMeasureByTableName(table.getTableName());
allCols.addAll(msrs);
Set<String>[] set = new HashSet[dims.size()];
for (int i = 0; i < set.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index 8e6abd4..f72bb7a 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -363,7 +363,7 @@ public class CarbonTableReader {
.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier(), null).getPath();
config.set(CarbonTableInputFormat.INPUT_DIR, carbonTablePath);
config.set(CarbonTableInputFormat.DATABASE_NAME, carbonTable.getDatabaseName());
- config.set(CarbonTableInputFormat.TABLE_NAME, carbonTable.getFactTableName());
+ config.set(CarbonTableInputFormat.TABLE_NAME, carbonTable.getTableName());
try {
CarbonTableInputFormat.setTableInfo(config, tableInfo);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index 17a4188..1430baf 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -333,10 +333,10 @@ object CarbonDataStoreCreator {
val split: Array[String] = header.split(",")
val allCols: util.List[CarbonColumn] = new util.ArrayList[CarbonColumn]()
val dims: util.List[CarbonDimension] =
- table.getDimensionByTableName(table.getFactTableName)
+ table.getDimensionByTableName(table.getTableName)
allCols.addAll(dims)
val msrs: List[CarbonMeasure] =
- table.getMeasureByTableName(table.getFactTableName)
+ table.getMeasureByTableName(table.getTableName)
allCols.addAll(msrs)
val set: Array[util.Set[String]] = Array.ofDim[util.Set[String]](dims.size)
for (i <- set.indices) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 6b435c6..1d41664 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -147,7 +147,7 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
case logicalRelation:LogicalRelation =>
if(logicalRelation.relation.isInstanceOf[CarbonDatasourceHadoopRelation]) {
val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- if(relation.carbonTable.getFactTableName.equalsIgnoreCase(actualTableName)) {
+ if(relation.carbonTable.getTableName.equalsIgnoreCase(actualTableName)) {
isValidPlan = true
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
index 3f99922..df1bd2e 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
@@ -51,7 +51,7 @@ class TestDDLForPartitionTable extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_hashTable")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
assert(partitionInfo != null)
assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("empno"))
assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataTypes.INT)
@@ -74,7 +74,7 @@ class TestDDLForPartitionTable extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_rangeTable")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
assert(partitionInfo != null)
assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("doj"))
assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataTypes.TIMESTAMP)
@@ -101,7 +101,7 @@ class TestDDLForPartitionTable extends QueryTest with BeforeAndAfterAll {
| 'LIST_INFO'='0, 1, (2, 3)')
""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_listTable")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
assert(partitionInfo != null)
assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("workgroupcategory"))
assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataTypes.STRING)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTableWithDefaultProperties.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTableWithDefaultProperties.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTableWithDefaultProperties.scala
index 317e2e2..c17ca00 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTableWithDefaultProperties.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTableWithDefaultProperties.scala
@@ -45,7 +45,7 @@ class TestDDLForPartitionTableWithDefaultProperties extends QueryTest with Befo
""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_hashTable")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
assert(partitionInfo != null)
assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("empno"))
assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataTypes.INT)
@@ -68,7 +68,7 @@ class TestDDLForPartitionTableWithDefaultProperties extends QueryTest with Befo
""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_rangeTable")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
assert(partitionInfo != null)
assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("doj"))
assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataTypes.TIMESTAMP)
@@ -96,7 +96,7 @@ class TestDDLForPartitionTableWithDefaultProperties extends QueryTest with Befo
| 'DICTIONARY_INCLUDE'='projectenddate')
""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_listTable")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
assert(partitionInfo != null)
assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("projectenddate"))
assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataTypes.TIMESTAMP)
@@ -128,7 +128,7 @@ class TestDDLForPartitionTableWithDefaultProperties extends QueryTest with Befo
| 'LIST_INFO'='2017-06-11 , 2017-06-13')
""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_listTableDate")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
assert(partitionInfo != null)
assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("projectenddate"))
assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataTypes.DATE)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
index 8eb5101..51e0cc4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
@@ -51,8 +51,8 @@ object ValidateUtil {
def validateSortScope(carbonTable: CarbonTable, sortScope: String): Unit = {
if (sortScope != null) {
// Don't support use global sort on partitioned table.
- if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null &&
- sortScope.equalsIgnoreCase(SortScopeOptions.SortScope.GLOBAL_SORT.toString)) {
+ if (carbonTable.getPartitionInfo(carbonTable.getTableName) != null &&
+ sortScope.equalsIgnoreCase(SortScopeOptions.SortScope.GLOBAL_SORT.toString)) {
throw new MalformedCarbonCommandException("Don't support use global sort on partitioned " +
"table.")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
index 5c6760a..37ab8c3 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
@@ -46,7 +46,7 @@ class AlterTableLoadPartitionRDD[K, V](alterPartitionModel: AlterPartitionModel,
val oldPartitionIds = alterPartitionModel.oldPartitionIds
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val databaseName = carbonTable.getDatabaseName
- val factTableName = carbonTable.getFactTableName
+ val factTableName = carbonTable.getTableName
val partitionInfo = carbonTable.getPartitionInfo(factTableName)
override protected def getPartitions: Array[Partition] = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 9ca21bc..0fed5a7 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -548,7 +548,7 @@ class PartitionTableDataLoaderRDD[K, V](
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
val model: CarbonLoadModel = carbonLoadModel
val carbonTable = model.getCarbonDataLoadSchema.getCarbonTable
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val uniqueLoadStatusId =
carbonLoadModel.getTableName + CarbonCommonConstants.UNDERSCORE + theSplit.index
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
index a82ea00..2aa5610 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
@@ -41,7 +41,7 @@ object PartitionDropper {
val dropWithData = dropPartitionCallableModel.dropWithData
val carbonTable = dropPartitionCallableModel.carbonTable
val dbName = carbonTable.getDatabaseName
- val tableName = carbonTable.getFactTableName
+ val tableName = carbonTable.getTableName
val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
val partitionInfo = carbonTable.getPartitionInfo(tableName)
val partitioner = PartitionFactory.getPartitioner(partitionInfo)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala
index db664b3..9106cca 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala
@@ -40,7 +40,7 @@ object PartitionSplitter {
val carbonLoadModel = splitPartitionCallableModel.carbonLoadModel
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
- val tableName = carbonTable.getFactTableName
+ val tableName = carbonTable.getTableName
val databaseName = carbonTable.getDatabaseName
val bucketInfo = carbonTable.getBucketingInfo(tableName)
var finalSplitStatus = false
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 1b21e3d..a3572ed 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -55,7 +55,6 @@ import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
-import org.apache.carbondata.processing.merger.TableMeta
import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil, CarbonLoaderUtil}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.rdd.CarbonMergeFilesRDD
@@ -516,7 +515,6 @@ object CommonUtil {
}
def readAndUpdateLoadProgressInTableMeta(model: CarbonLoadModel,
- storePath: String,
insertOverwrite: Boolean): Unit = {
val newLoadMetaEntry = new LoadMetadataDetails
val status = if (insertOverwrite) {
@@ -528,16 +526,13 @@ object CommonUtil {
// reading the start time of data load.
val loadStartTime = CarbonUpdateUtil.readCurrentTime
model.setFactTimeStamp(loadStartTime)
- CarbonLoaderUtil
- .populateNewLoadMetaEntry(newLoadMetaEntry, status, model.getFactTimeStamp, false)
+ CarbonLoaderUtil.populateNewLoadMetaEntry(
+ newLoadMetaEntry, status, model.getFactTimeStamp, false)
val entryAdded: Boolean =
CarbonLoaderUtil.recordLoadMetadata(newLoadMetaEntry, model, true, insertOverwrite)
if (!entryAdded) {
- sys
- .error(s"Failed to add entry in table status for ${ model.getDatabaseName }.${
- model
- .getTableName
- }")
+ sys.error(s"Failed to add entry in table status for " +
+ s"${ model.getDatabaseName }.${model.getTableName}")
}
}
@@ -856,26 +851,9 @@ object CommonUtil {
CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT,
CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT).toBoolean) {
new CarbonMergeFilesRDD(sparkContext, AbsoluteTableIdentifier.from(tablePath,
- carbonTable.getDatabaseName, carbonTable.getFactTableName).getTablePath,
+ carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
segmentIds).collect()
}
}
- /**
- * can be removed with the spark 1.6 removal
- * @param tableMeta
- * @return
- */
- @deprecated
- def getTablePath(tableMeta: TableMeta): String = {
- if (tableMeta.tablePath == null) {
- tableMeta.storePath + CarbonCommonConstants.FILE_SEPARATOR +
- tableMeta.carbonTableIdentifier.getDatabaseName +
- CarbonCommonConstants.FILE_SEPARATOR + tableMeta.carbonTableIdentifier.getTableName
- }
- else {
- tableMeta.tablePath
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 35e1e78..84ad85e 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -180,10 +180,10 @@ object DataLoadingUtil {
options: immutable.Map[String, String],
optionsFinal: mutable.Map[String, String],
carbonLoadModel: CarbonLoadModel): Unit = {
- carbonLoadModel.setTableName(table.getFactTableName)
+ carbonLoadModel.setTableName(table.getTableName)
carbonLoadModel.setDatabaseName(table.getDatabaseName)
carbonLoadModel.setTablePath(table.getTablePath)
- carbonLoadModel.setTableName(table.getFactTableName)
+ carbonLoadModel.setTableName(table.getTableName)
val dataLoadSchema = new CarbonDataLoadSchema(table)
// Need to fill dimension relation
carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
@@ -199,7 +199,7 @@ object DataLoadingUtil {
val complex_delimeter_level2 = optionsFinal("complex_delimiter_level_2")
val all_dictionary_path = optionsFinal("all_dictionary_path")
val column_dict = optionsFinal("columndict")
- ValidateUtil.validateDateFormat(dateFormat, table, table.getFactTableName)
+ ValidateUtil.validateDateFormat(dateFormat, table, table.getTableName)
ValidateUtil.validateSortScope(table, sort_scope)
if (bad_records_logger_enable.toBoolean ||
@@ -236,7 +236,7 @@ object DataLoadingUtil {
}
} else {
if (fileHeader.isEmpty) {
- fileHeader = table.getCreateOrderColumn(table.getFactTableName)
+ fileHeader = table.getCreateOrderColumn(table.getTableName)
.asScala.map(_.getColName).mkString(",")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 975fc9b..0bf2b16 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -676,17 +676,17 @@ object GlobalDictionaryUtil {
*/
def generateGlobalDictionary(sqlContext: SQLContext,
carbonLoadModel: CarbonLoadModel,
- storePath: String,
+ tablePath: String,
dataFrame: Option[DataFrame] = None): Unit = {
try {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val carbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier
// create dictionary folder if not exists
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier)
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(tablePath, carbonTableIdentifier)
val dictfolderPath = carbonTablePath.getMetadataDirectoryPath
// columns which need to generate global dictionary file
val dimensions = carbonTable.getDimensionByTableName(
- carbonTable.getFactTableName).asScala.toArray
+ carbonTable.getTableName).asScala.toArray
// generate global dict from pre defined column dict file
carbonLoadModel.initPredefDictMap()
@@ -701,7 +701,7 @@ object GlobalDictionaryUtil {
if (colDictFilePath != null) {
// generate predefined dictionary
generatePredefinedColDictionary(colDictFilePath, carbonTableIdentifier,
- dimensions, carbonLoadModel, sqlContext, storePath, dictfolderPath)
+ dimensions, carbonLoadModel, sqlContext, tablePath, dictfolderPath)
}
if (headers.length > df.columns.length) {
val msg = "The number of columns in the file header do not match the " +
@@ -717,7 +717,7 @@ object GlobalDictionaryUtil {
// select column to push down pruning
df = df.select(requireColumnNames.head, requireColumnNames.tail: _*)
val model = createDictionaryLoadModel(carbonLoadModel, carbonTableIdentifier,
- requireDimension, storePath, dictfolderPath, false)
+ requireDimension, tablePath, dictfolderPath, false)
// combine distinct value in a block and partition by column
val inputRDD = new CarbonBlockDistinctValuesCombineRDD(df.rdd, model)
.partitionBy(new ColumnPartitioner(model.primDimensions.length))
@@ -731,7 +731,7 @@ object GlobalDictionaryUtil {
} else {
generateDictionaryFromDictionaryFiles(sqlContext,
carbonLoadModel,
- storePath,
+ tablePath,
carbonTableIdentifier,
dictfolderPath,
dimensions,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 756de6b..2f6b277 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -454,7 +454,7 @@ class TableNewProcessor(cm: TableModel) {
val field = cm.dimCols.find(keyDim equals _.column).get
val encoders = if (cm.parentTable.isDefined && cm.dataMapRelation.get.get(field).isDefined) {
cm.parentTable.get.getColumnByName(
- cm.parentTable.get.getFactTableName,
+ cm.parentTable.get.getTableName,
cm.dataMapRelation.get.get(field).get.columnTableRelation.get.parentColumnName).getEncoder
} else {
val encoders = new java.util.ArrayList[Encoding]()
@@ -479,7 +479,7 @@ class TableNewProcessor(cm: TableModel) {
val encoders = if (cm.parentTable.isDefined &&
cm.dataMapRelation.get.get(field).isDefined) {
cm.parentTable.get.getColumnByName(
- cm.parentTable.get.getFactTableName,
+ cm.parentTable.get.getTableName,
cm.dataMapRelation.get.get(field).get.
columnTableRelation.get.parentColumnName).getEncoder
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 1ca7456..c12d2ef 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -179,7 +179,7 @@ object CarbonDataRDDFactory {
while (null != tableForCompaction) {
LOGGER.info("Compaction request has been identified for table " +
s"${ tableForCompaction.getDatabaseName }." +
- s"${ tableForCompaction.getFactTableName}")
+ s"${ tableForCompaction.getTableName}")
val table: CarbonTable = tableForCompaction
val metadataPath = table.getMetaDataFilepath
val compactionType = CarbonCompactionUtil.determineCompactionType(metadataPath)
@@ -204,7 +204,7 @@ object CarbonDataRDDFactory {
case e: Exception =>
LOGGER.error("Exception in compaction thread for table " +
s"${ tableForCompaction.getDatabaseName }." +
- s"${ tableForCompaction.getFactTableName }")
+ s"${ tableForCompaction.getTableName }")
// not handling the exception. only logging as this is not the table triggered
// by user.
} finally {
@@ -216,7 +216,7 @@ object CarbonDataRDDFactory {
skipCompactionTables.+=:(tableForCompaction.getCarbonTableIdentifier)
LOGGER.error("Compaction request file can not be deleted for table " +
s"${ tableForCompaction.getDatabaseName }." +
- s"${ tableForCompaction.getFactTableName }")
+ s"${ tableForCompaction.getTableName }")
}
}
// ********* check again for all the tables.
@@ -248,7 +248,7 @@ object CarbonDataRDDFactory {
table: CarbonTable
): CarbonLoadModel = {
val loadModel = new CarbonLoadModel
- loadModel.setTableName(table.getFactTableName)
+ loadModel.setTableName(table.getTableName)
val dataLoadSchema = new CarbonDataLoadSchema(table)
// Need to fill dimension relation
loadModel.setCarbonDataLoadSchema(dataLoadSchema)
@@ -319,7 +319,7 @@ object CarbonDataRDDFactory {
}
}
} else {
- status = if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null) {
+ status = if (carbonTable.getPartitionInfo(carbonTable.getTableName) != null) {
loadDataForPartitionTable(sqlContext, dataFrame, carbonLoadModel)
} else if (isSortTable && sortScope.equals(SortScopeOptions.SortScope.GLOBAL_SORT)) {
DataLoadProcessBuilderOnSpark.loadDataUsingGlobalSort(sqlContext.sparkContext,
@@ -782,7 +782,7 @@ object CarbonDataRDDFactory {
dataFrame: Option[DataFrame],
carbonLoadModel: CarbonLoadModel): RDD[Row] = {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val partitionColumn = partitionInfo.getColumnSchemaList.get(0).getColumnName
val partitionColumnDataType = partitionInfo.getColumnSchemaList.get(0).getDataType
val columns = carbonLoadModel.getCsvHeaderColumns
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
index 1e6a36e..47f5344 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
@@ -21,23 +21,20 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.hive.{CarbonMetaData, CarbonRelation, DictionaryMap}
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
-import org.apache.carbondata.core.util.CarbonUtil
-import org.apache.carbondata.processing.merger.TableMeta
case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
object CarbonSparkUtil {
def createSparkMeta(carbonTable: CarbonTable): CarbonMetaData = {
- val dimensionsAttr = carbonTable.getDimensionByTableName(carbonTable.getFactTableName)
+ val dimensionsAttr = carbonTable.getDimensionByTableName(carbonTable.getTableName)
.asScala.map(x => x.getColName) // wf : may be problem
- val measureAttr = carbonTable.getMeasureByTableName(carbonTable.getFactTableName)
+ val measureAttr = carbonTable.getMeasureByTableName(carbonTable.getTableName)
.asScala.map(x => x.getColName)
val dictionary =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName).asScala.map { f =>
+ carbonTable.getDimensionByTableName(carbonTable.getTableName).asScala.map { f =>
(f.getColName.toLowerCase,
f.hasEncoding(Encoding.DICTIONARY) && !f.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
!f.getDataType.isComplexType)
@@ -47,10 +44,11 @@ object CarbonSparkUtil {
def createCarbonRelation(tableInfo: TableInfo, tablePath: String): CarbonRelation = {
val table = CarbonTable.buildFromTableInfo(tableInfo)
- val meta = new TableMeta(table.getCarbonTableIdentifier,
- table.getTablePath, tablePath, table)
- CarbonRelation(tableInfo.getDatabaseName, tableInfo.getFactTable.getTableName,
- CarbonSparkUtil.createSparkMeta(table), meta)
+ CarbonRelation(
+ tableInfo.getDatabaseName,
+ tableInfo.getFactTable.getTableName,
+ CarbonSparkUtil.createSparkMeta(table),
+ table)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 44fbb37..b74576d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.types._
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonType}
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.CarbonOption
class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame) {
@@ -58,7 +59,7 @@ class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame) {
*/
private def loadTempCSV(options: CarbonOption): Unit = {
// temporary solution: write to csv file, then load the csv into carbon
- val storePath = CarbonEnv.getInstance(sqlContext.sparkSession).storePath
+ val storePath = CarbonProperties.getStorePath
val tempCSVFolder = new StringBuilder(storePath).append(CarbonCommonConstants.FILE_SEPARATOR)
.append("tempCSV")
.append(CarbonCommonConstants.UNDERSCORE).append(options.dbName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 22933f2..72f40ac 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -50,7 +50,7 @@ case class CarbonDatasourceHadoopRelation(
lazy val identifier: AbsoluteTableIdentifier = AbsoluteTableIdentifier.from(paths.head,
parameters("dbname"), parameters("tablename"))
lazy val databaseName: String = carbonTable.getDatabaseName
- lazy val tableName: String = carbonTable.getFactTableName
+ lazy val tableName: String = carbonTable.getTableName
CarbonSession.updateSessionInfoToCurrentThread(sparkSession)
@transient lazy val carbonRelation: CarbonRelation =
@@ -58,7 +58,7 @@ case class CarbonDatasourceHadoopRelation(
createCarbonRelation(parameters, identifier, sparkSession)
- @transient lazy val carbonTable: CarbonTable = carbonRelation.tableMeta.carbonTable
+ @transient lazy val carbonTable: CarbonTable = carbonRelation.carbonTable
override def sqlContext: SQLContext = sparkSession.sqlContext
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index c7db436..9d88c4c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -72,7 +72,7 @@ case class CarbonDictionaryDecoder(
attachTree(this, "execute") {
val absoluteTableIdentifiers = relations.map { relation =>
val carbonTable = relation.carbonRelation.carbonRelation.metaData.carbonTable
- (carbonTable.getFactTableName, carbonTable.getAbsoluteTableIdentifier)
+ (carbonTable.getTableName, carbonTable.getAbsoluteTableIdentifier)
}.toMap
if (CarbonDictionaryDecoder.isRequiredToDecode(getDictionaryColumnIds)) {
@@ -125,7 +125,7 @@ case class CarbonDictionaryDecoder(
val absoluteTableIdentifiers = relations.map { relation =>
val carbonTable = relation.carbonRelation.carbonRelation.metaData.carbonTable
- (carbonTable.getFactTableName, carbonTable.getAbsoluteTableIdentifier)
+ (carbonTable.getTableName, carbonTable.getAbsoluteTableIdentifier)
}.toMap
if (CarbonDictionaryDecoder.isRequiredToDecode(getDictionaryColumnIds)) {
@@ -323,7 +323,7 @@ object CarbonDictionaryDecoder {
if (relation.isDefined && canBeDecoded(attr, profile)) {
val carbonTable = relation.get.carbonRelation.carbonRelation.metaData.carbonTable
val carbonDimension = carbonTable
- .getDimensionByName(carbonTable.getFactTableName, attr.name)
+ .getDimensionByName(carbonTable.getTableName, attr.name)
if (carbonDimension != null &&
carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
!carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
@@ -355,7 +355,7 @@ object CarbonDictionaryDecoder {
if (relation.isDefined) {
val carbonTable = relation.get.carbonRelation.carbonRelation.metaData.carbonTable
val carbonDimension = carbonTable
- .getDimensionByName(carbonTable.getFactTableName, attr.name)
+ .getDimensionByName(carbonTable.getTableName, attr.name)
if (carbonDimension != null &&
carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
!carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
@@ -432,12 +432,12 @@ object CarbonDictionaryDecoder {
if (relation.isDefined && CarbonDictionaryDecoder.canBeDecoded(attr, profile)) {
val carbonTable = relation.get.carbonRelation.carbonRelation.metaData.carbonTable
val carbonDimension =
- carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
+ carbonTable.getDimensionByName(carbonTable.getTableName, attr.name)
if (carbonDimension != null &&
carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
!carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
!carbonDimension.isComplex) {
- (carbonTable.getFactTableName, carbonDimension.getColumnIdentifier,
+ (carbonTable.getTableName, carbonDimension.getColumnIdentifier,
carbonDimension)
} else {
(null, null, null)
@@ -485,12 +485,12 @@ class CarbonDecoderRDD(
if (relation.isDefined && canBeDecoded(attr)) {
val carbonTable = relation.get.carbonRelation.carbonRelation.metaData.carbonTable
val carbonDimension =
- carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
+ carbonTable.getDimensionByName(carbonTable.getTableName, attr.name)
if (carbonDimension != null &&
carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
!carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY) &&
!carbonDimension.isComplex()) {
- (carbonTable.getFactTableName, carbonDimension.getColumnIdentifier,
+ (carbonTable.getTableName, carbonDimension.getColumnIdentifier,
carbonDimension)
} else {
(null, null, null)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 1ee7650..dcfce0f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -20,13 +20,15 @@ package org.apache.spark.sql
import java.util.Map
import java.util.concurrent.ConcurrentHashMap
-import org.apache.spark.sql.hive.{CarbonMetaStore, CarbonMetaStoreFactory, CarbonSessionCatalog}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.hive.{CarbonMetaStore, CarbonMetaStoreFactory, CarbonRelation, CarbonSessionCatalog}
import org.apache.spark.sql.internal.CarbonSQLConf
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, SessionParams, ThreadLocalSessionInfo}
-import org.apache.carbondata.events.{CarbonEnvInitPreEvent, OperationContext, OperationListenerBus}
+import org.apache.carbondata.events.{CarbonEnvInitPreEvent, OperationListenerBus}
import org.apache.carbondata.spark.rdd.SparkReadSupport
import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
@@ -41,8 +43,6 @@ class CarbonEnv {
var carbonSessionInfo: CarbonSessionInfo = _
- var storePath: String = _
-
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
// set readsupport class global so that the executor can get it.
@@ -74,7 +74,7 @@ class CarbonEnv {
config.addDefaultCarbonSessionParams()
carbonMetastore = {
val properties = CarbonProperties.getInstance()
- storePath = properties.getProperty(CarbonCommonConstants.STORE_LOCATION)
+ var storePath = properties.getProperty(CarbonCommonConstants.STORE_LOCATION)
if (storePath == null) {
storePath = sparkSession.conf.get("spark.sql.warehouse.dir")
properties.addProperty(CarbonCommonConstants.STORE_LOCATION, storePath)
@@ -112,6 +112,30 @@ object CarbonEnv {
carbonEnv
}
}
-}
+ /**
+ * Return carbon table instance by looking up relation in `sparkSession`
+ */
+ def getCarbonTable(
+ databaseNameOp: Option[String],
+ tableName: String)
+ (sparkSession: SparkSession): CarbonTable = {
+ CarbonEnv
+ .getInstance(sparkSession)
+ .carbonMetastore
+ .lookupRelation(databaseNameOp, tableName)(sparkSession)
+ .asInstanceOf[CarbonRelation]
+ .carbonTable
+ }
+ def getCarbonTable(
+ tableIdentifier: TableIdentifier)
+ (sparkSession: SparkSession): CarbonTable = {
+ CarbonEnv
+ .getInstance(sparkSession)
+ .carbonMetastore
+ .lookupRelation(tableIdentifier)(sparkSession)
+ .asInstanceOf[CarbonRelation]
+ .carbonTable
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index 0806421..99a7c37 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -65,11 +65,11 @@ case class CarbonScan(
attributesRaw = attributeOut
}
- val columns = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName)
+ val columns = carbonTable.getCreateOrderColumn(carbonTable.getTableName)
val colAttr = new Array[Attribute](columns.size())
attributesRaw.foreach { attr =>
val column =
- carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name)
+ carbonTable.getColumnByName(carbonTable.getTableName, attr.name)
if(column != null) {
colAttr(columns.indexOf(column)) = attr
}
@@ -78,7 +78,7 @@ case class CarbonScan(
var queryOrder: Integer = 0
attributesRaw.foreach { attr =>
- val carbonColumn = carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name)
+ val carbonColumn = carbonTable.getColumnByName(carbonTable.getTableName, attr.name)
if (carbonColumn != null) {
if (carbonColumn.isDimension()) {
val dim = new QueryDimension(attr.name)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index fba590e..6331f12 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -165,7 +165,7 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
} else {
CarbonEnv.getInstance(sparkSession).carbonMetastore
.lookupRelation(Option(dbName), tableName)(sparkSession)
- (CarbonEnv.getInstance(sparkSession).storePath + s"/$dbName/$tableName", parameters)
+ (CarbonProperties.getStorePath + s"/$dbName/$tableName", parameters)
}
} catch {
case ex: NoSuchTableException =>
@@ -199,11 +199,10 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
if (parameters.contains("tablePath")) {
(parameters("tablePath"), parameters)
} else if (!sparkSession.isInstanceOf[CarbonSession]) {
- (CarbonEnv.getInstance(sparkSession).storePath + "/" + dbName + "/" + tableName, parameters)
+ (CarbonProperties.getStorePath + "/" + dbName + "/" + tableName, parameters)
} else {
- val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
- (relation.tableMeta.tablePath, parameters)
+ val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
+ (carbonTable.getTablePath, parameters)
}
} catch {
case ex: Exception =>
@@ -235,15 +234,11 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
}
if (tablePathOption.isDefined) {
val sparkSession = sqlContext.sparkSession
- val identifier: AbsoluteTableIdentifier =
- AbsoluteTableIdentifier.from(tablePathOption.get, dbName, tableName)
- val carbonTable =
- CarbonEnv.getInstance(sparkSession).carbonMetastore.
- createCarbonRelation(parameters, identifier, sparkSession).tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
if (!carbonTable.isStreamingTable) {
throw new CarbonStreamException(s"Table ${carbonTable.getDatabaseName}." +
- s"${carbonTable.getFactTableName} is not a streaming table")
+ s"${carbonTable.getTableName} is not a streaming table")
}
// create sink
@@ -314,8 +309,7 @@ object CarbonSource {
val tableName: String = properties.getOrElse("tableName", "").toLowerCase
val model = createTableInfoFromParams(properties, dataSchema, dbName, tableName)
val tableInfo: TableInfo = TableNewProcessor(model)
- val dbLocation = GetDB.getDatabaseLocation(dbName, sparkSession,
- CarbonEnv.getInstance(sparkSession).storePath)
+ val dbLocation = GetDB.getDatabaseLocation(dbName, sparkSession, CarbonProperties.getStorePath)
val tablePath = dbLocation + CarbonCommonConstants.FILE_SEPARATOR + tableName
val schemaEvolutionEntry = new SchemaEvolutionEntry
schemaEvolutionEntry.setTimeStamp(tableInfo.getLastUpdatedTime)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonCreateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonCreateTableCommand.scala
index 197b23b..f83766d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonCreateTableCommand.scala
@@ -25,7 +25,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.exception.InvalidConfigurationException
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.TableInfo
-import org.apache.carbondata.core.util.CarbonUtil
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
case class CarbonCreateTableCommand(
cm: TableModel,
@@ -37,7 +37,7 @@ case class CarbonCreateTableCommand(
}
override def processSchema(sparkSession: SparkSession): Seq[Row] = {
- val storePath = CarbonEnv.getInstance(sparkSession).storePath
+ val storePath = CarbonProperties.getStorePath
CarbonEnv.getInstance(sparkSession).carbonMetastore.
checkSchemasModifiedTimeAndReloadTables()
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
index 7dcad9a..b233c99 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
@@ -29,6 +29,7 @@ import org.codehaus.jackson.map.ObjectMapper
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.util.CarbonProperties
private[sql] case class CarbonDescribeFormattedCommand(
child: SparkPlan,
@@ -68,7 +69,7 @@ private[sql] case class CarbonDescribeFormattedCommand(
val colComment = field.getComment().getOrElse("null")
val comment = if (dims.contains(fieldName)) {
val dimension = relation.metaData.carbonTable.getDimensionByName(
- relation.tableMeta.carbonTableIdentifier.getTableName, fieldName)
+ relation.carbonTable.getTableName, fieldName)
if (null != dimension.getColumnProperties && !dimension.getColumnProperties.isEmpty) {
colProps.append(fieldName).append(".")
.append(mapper.writeValueAsString(dimension.getColumnProperties))
@@ -101,12 +102,11 @@ private[sql] case class CarbonDescribeFormattedCommand(
colProps.toString()
}
results ++= Seq(("", "", ""), ("##Detailed Table Information", "", ""))
- results ++= Seq(("Database Name: ", relation.tableMeta.carbonTableIdentifier
- .getDatabaseName, "")
+ results ++= Seq(("Database Name: ", relation.carbonTable.getDatabaseName, "")
)
- results ++= Seq(("Table Name: ", relation.tableMeta.carbonTableIdentifier.getTableName, ""))
- results ++= Seq(("CARBON Store Path: ", relation.tableMeta.storePath, ""))
- val carbonTable = relation.tableMeta.carbonTable
+ results ++= Seq(("Table Name: ", relation.carbonTable.getTableName, ""))
+ results ++= Seq(("CARBON Store Path: ", CarbonProperties.getStorePath, ""))
+ val carbonTable = relation.carbonTable
// Carbon table support table comment
val tableComment = carbonTable.getTableInfo.getFactTable.getTableProperties
.getOrDefault(CarbonCommonConstants.TABLE_COMMENT, "")
@@ -122,14 +122,14 @@ private[sql] case class CarbonDescribeFormattedCommand(
results ++= Seq(("ADAPTIVE", "", ""))
}
results ++= Seq(("SORT_COLUMNS", relation.metaData.carbonTable.getSortColumns(
- relation.tableMeta.carbonTableIdentifier.getTableName).asScala
+ relation.carbonTable.getTableName).asScala
.map(column => column).mkString(","), ""))
val dimension = carbonTable
- .getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
+ .getDimensionByTableName(relation.carbonTable.getTableName)
results ++= getColumnGroups(dimension.asScala.toList)
- if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null) {
+ if (carbonTable.getPartitionInfo(carbonTable.getTableName) != null) {
results ++=
- Seq(("Partition Columns: ", carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ Seq(("Partition Columns: ", carbonTable.getPartitionInfo(carbonTable.getTableName)
.getColumnSchemaList.asScala.map(_.getColumnName).mkString(","), ""))
}
results.map {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDropTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDropTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDropTableCommand.scala
index 0343393..f0a916a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDropTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDropTableCommand.scala
@@ -30,7 +30,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.CarbonUtil
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.events._
@@ -50,12 +50,11 @@ case class CarbonDropTableCommand(
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
val dbName = GetDB.getDatabaseName(databaseNameOp, sparkSession)
val identifier = TableIdentifier(tableName, Option(dbName))
- val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
val locksToBeAcquired = List(LockUsage.METADATA_LOCK, LockUsage.DROP_TABLE_LOCK)
val carbonEnv = CarbonEnv.getInstance(sparkSession)
val catalog = carbonEnv.carbonMetastore
val databaseLocation = GetDB.getDatabaseLocation(dbName, sparkSession,
- CarbonEnv.getInstance(sparkSession).storePath)
+ CarbonProperties.getStorePath)
val tablePath = databaseLocation + CarbonCommonConstants.FILE_SEPARATOR + tableName.toLowerCase
val absoluteTableIdentifier = AbsoluteTableIdentifier
.from(tablePath, dbName.toLowerCase, tableName.toLowerCase)
@@ -68,7 +67,7 @@ case class CarbonDropTableCommand(
LOGGER.audit(s"Deleting table [$tableName] under database [$dbName]")
val carbonTable: Option[CarbonTable] =
catalog.getTableFromMetadataCache(dbName, tableName) match {
- case Some(tableMeta) => Some(tableMeta.carbonTable)
+ case Some(carbonTable) => Some(carbonTable)
case None => try {
Some(catalog.lookupRelation(identifier)(sparkSession)
.asInstanceOf[CarbonRelation].metaData.carbonTable)
@@ -131,7 +130,7 @@ case class CarbonDropTableCommand(
// delete the table folder
val dbName = GetDB.getDatabaseName(databaseNameOp, sparkSession)
val databaseLocation = GetDB.getDatabaseLocation(dbName, sparkSession,
- CarbonEnv.getInstance(sparkSession).storePath)
+ CarbonProperties.getStorePath)
val tablePath = databaseLocation + CarbonCommonConstants.FILE_SEPARATOR + tableName.toLowerCase
val tableIdentifier = AbsoluteTableIdentifier.from(tablePath, dbName, tableName)
val metadataFilePath =
[05/28] carbondata git commit: [CARBONDATA-1733] While load is in
progress, Show segments is throwing NPE
Posted by ja...@apache.org.
[CARBONDATA-1733] While load is in progress, Show segments is throwing NPE
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/aff3b9e4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/aff3b9e4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/aff3b9e4
Branch: refs/heads/fgdatamap
Commit: aff3b9e4c033ac736de4888195a85906ebd61b13
Parents: 808a334
Author: dhatchayani <dh...@gmail.com>
Authored: Thu Nov 16 16:15:38 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Thu Nov 16 20:47:01 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/statusmanager/LoadMetadataDetails.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/aff3b9e4/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index f42ca23..d838e2e 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -202,6 +202,10 @@ public class LoadMetadataDetails implements Serializable {
LOGGER.error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e.getMessage());
parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
try {
+ // if the load is in progress, factTimeStamp will be null, so use current time
+ if (null == factTimeStamp) {
+ return System.currentTimeMillis();
+ }
dateToStr = parser.parse(factTimeStamp);
return dateToStr.getTime();
} catch (ParseException e1) {
[08/28] carbondata git commit: [CARBONDATA-1717]Remove spark
broadcast for gettting hadoop configurations
Posted by ja...@apache.org.
[CARBONDATA-1717]Remove spark broadcast for gettting hadoop configurations
This closes #1500
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b6777fcc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b6777fcc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b6777fcc
Branch: refs/heads/fgdatamap
Commit: b6777fcc32df3ce3616ea02f5566ab5bf4ca6e30
Parents: 733bb51
Author: akashrn5 <ak...@gmail.com>
Authored: Fri Oct 27 18:11:03 2017 +0530
Committer: QiangCai <qi...@qq.com>
Committed: Fri Nov 17 14:33:10 2017 +0800
----------------------------------------------------------------------
.../spark/rdd/NewCarbonDataLoadRDD.scala | 26 ++++++++++++++++----
1 file changed, 21 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6777fcc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 6f44a0d..9ca21bc 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -17,7 +17,7 @@
package org.apache.carbondata.spark.rdd
-import java.io.{File, IOException, ObjectInputStream, ObjectOutputStream}
+import java.io._
import java.nio.ByteBuffer
import java.text.SimpleDateFormat
import java.util.{Date, UUID}
@@ -41,7 +41,9 @@ import org.apache.carbondata.common.CarbonIterator
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.compression.CompressorFactory
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
import org.apache.carbondata.processing.loading.{DataLoadExecutor, FailureCauses}
import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat, CSVRecordReaderIterator}
@@ -187,9 +189,23 @@ class NewCarbonDataLoadRDD[K, V](
formatter.format(new Date())
}
- // A Hadoop Configuration can be about 10 KB, which is pretty big, so broadcast it
- private val confBroadcast =
- sc.broadcast(new SerializableConfiguration(sc.hadoopConfiguration))
+ private val confBytes = {
+ val bao = new ByteArrayOutputStream()
+ val oos = new ObjectOutputStream(bao)
+ sc.hadoopConfiguration.write(oos)
+ oos.close()
+ CompressorFactory.getInstance().getCompressor.compressByte(bao.toByteArray)
+ }
+
+ private def getConf = {
+ val configuration = new Configuration(false)
+ val bai = new ByteArrayInputStream(CompressorFactory.getInstance().getCompressor
+ .unCompressByte(confBytes))
+ val ois = new ObjectInputStream(bai)
+ configuration.readFields(ois)
+ ois.close()
+ configuration
+ }
override def getPartitions: Array[Partition] = {
blocksGroupBy.zipWithIndex.map { b =>
@@ -255,7 +271,7 @@ class NewCarbonDataLoadRDD[K, V](
def getInputIterators: Array[CarbonIterator[Array[AnyRef]]] = {
val attemptId = new TaskAttemptID(jobTrackerId, id, TaskType.MAP, theSplit.index, 0)
- var configuration: Configuration = confBroadcast.value.value
+ var configuration: Configuration = getConf
if (configuration == null) {
configuration = new Configuration()
}
[27/28] carbondata git commit: [CARBONDATA-1544][Datamap] Datamap
FineGrain implementation
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
index 78544d3..fe0bbcf 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.datamap.examples;
import java.io.BufferedWriter;
import java.io.DataOutputStream;
-import java.io.File;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
@@ -29,17 +28,18 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import com.google.gson.Gson;
-public class MinMaxDataWriter implements DataMapWriter {
+public class MinMaxDataWriter extends AbstractDataMapWriter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(TableInfo.class.getName());
@@ -50,17 +50,23 @@ public class MinMaxDataWriter implements DataMapWriter {
private Map<Integer, BlockletMinMax> blockMinMaxMap;
- private String blockPath;
+ private String dataWritePath;
+ public MinMaxDataWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String dataWritePath) {
+ super(identifier, segmentId, dataWritePath);
+ this.identifier = identifier;
+ this.segmentId = segmentId;
+ this.dataWritePath = dataWritePath;
+ }
- @Override public void onBlockStart(String blockId, String blockPath) {
+ @Override public void onBlockStart(String blockId) {
pageLevelMax = null;
pageLevelMin = null;
blockletLevelMax = null;
blockletLevelMin = null;
blockMinMaxMap = null;
blockMinMaxMap = new HashMap<Integer, BlockletMinMax>();
- this.blockPath = blockPath;
}
@Override public void onBlockEnd(String blockId) {
@@ -161,7 +167,7 @@ public class MinMaxDataWriter implements DataMapWriter {
List<MinMaxIndexBlockDetails> tempMinMaxIndexBlockDetails = null;
tempMinMaxIndexBlockDetails = loadBlockDetails();
try {
- writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockPath, blockId);
+ writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockId);
} catch (IOException ex) {
LOGGER.info(" Unable to write the file");
}
@@ -178,7 +184,6 @@ public class MinMaxDataWriter implements DataMapWriter {
tmpminMaxIndexBlockDetails.setMinValues(blockMinMaxMap.get(index).getMin());
tmpminMaxIndexBlockDetails.setMaxValues(blockMinMaxMap.get(index).getMax());
tmpminMaxIndexBlockDetails.setBlockletId(index);
- tmpminMaxIndexBlockDetails.setFilePath(this.blockPath);
minMaxIndexBlockDetails.add(tmpminMaxIndexBlockDetails);
}
return minMaxIndexBlockDetails;
@@ -187,22 +192,19 @@ public class MinMaxDataWriter implements DataMapWriter {
/**
* Write the data to a file. This is JSON format file.
* @param minMaxIndexBlockDetails
- * @param blockPath
* @param blockId
* @throws IOException
*/
public void writeMinMaxIndexFile(List<MinMaxIndexBlockDetails> minMaxIndexBlockDetails,
- String blockPath, String blockId) throws IOException {
- String filePath = blockPath.substring(0, blockPath.lastIndexOf(File.separator) + 1) + blockId
- + ".minmaxindex";
+ String blockId) throws IOException {
+ String filePath = dataWritePath +"/" + blockId + ".minmaxindex";
BufferedWriter brWriter = null;
DataOutputStream dataOutStream = null;
try {
FileFactory.createNewFile(filePath, FileFactory.getFileType(filePath));
dataOutStream = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath));
Gson gsonObjectToWrite = new Gson();
- brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream,
- CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT));
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream, "UTF-8"));
String minmaxIndexData = gsonObjectToWrite.toJson(minMaxIndexBlockDetails);
brWriter.write(minmaxIndexData);
} catch (IOException ioe) {
@@ -215,7 +217,11 @@ public class MinMaxDataWriter implements DataMapWriter {
dataOutStream.flush();
}
CarbonUtil.closeStreams(brWriter, dataOutStream);
+ commitFile(filePath);
}
}
+ @Override public void finish() throws IOException {
+
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
index 0596db5..93a453e 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
@@ -33,11 +33,6 @@ public class MinMaxIndexBlockDetails implements Serializable {
private byte[][] maxValues;
/**
- * filePath pointing to the block.
- */
- private String filePath;
-
- /**
* BlockletID of the block.
*/
private Integer BlockletId;
@@ -59,14 +54,6 @@ public class MinMaxIndexBlockDetails implements Serializable {
this.maxValues = maxValues;
}
- public String getFilePath() {
- return filePath;
- }
-
- public void setFilePath(String filePath) {
- this.filePath = filePath;
- }
-
public Integer getBlockletId() {
return BlockletId;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 88d8341..237434b 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -411,7 +411,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
}
carbonSplits.add(CarbonInputSplit.from(segmentId, fileSplit,
ColumnarFormatVersion.valueOf(
- CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION)));
+ CarbonCommonConstants.CARBON_DATA_FILE_DEFAULT_VERSION), null));
}
return carbonSplits;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index e89c2d6..3ac4642 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -84,6 +84,8 @@ public class CarbonInputSplit extends FileSplit
private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
+ private String dataMapWritePath;
+
public CarbonInputSplit() {
segmentId = null;
taskId = "0";
@@ -94,7 +96,7 @@ public class CarbonInputSplit extends FileSplit
}
private CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations,
- ColumnarFormatVersion version, String[] deleteDeltaFiles) {
+ ColumnarFormatVersion version, String[] deleteDeltaFiles, String dataMapWritePath) {
super(path, start, length, locations);
this.segmentId = segmentId;
String taskNo = CarbonTablePath.DataFileUtil.getTaskNo(path.getName());
@@ -106,11 +108,12 @@ public class CarbonInputSplit extends FileSplit
this.invalidSegments = new ArrayList<>();
this.version = version;
this.deleteDeltaFiles = deleteDeltaFiles;
+ this.dataMapWritePath = dataMapWritePath;
}
public CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations,
int numberOfBlocklets, ColumnarFormatVersion version, String[] deleteDeltaFiles) {
- this(segmentId, path, start, length, locations, version, deleteDeltaFiles);
+ this(segmentId, path, start, length, locations, version, deleteDeltaFiles, null);
this.numberOfBlocklets = numberOfBlocklets;
}
@@ -157,10 +160,10 @@ public class CarbonInputSplit extends FileSplit
}
public static CarbonInputSplit from(String segmentId, FileSplit split,
- ColumnarFormatVersion version)
+ ColumnarFormatVersion version, String dataMapWritePath)
throws IOException {
return new CarbonInputSplit(segmentId, split.getPath(), split.getStart(), split.getLength(),
- split.getLocations(), version, null);
+ split.getLocations(), version, null, dataMapWritePath);
}
public static List<TableBlockInfo> createBlocks(List<CarbonInputSplit> splitList) {
@@ -174,6 +177,7 @@ public class CarbonInputSplit extends FileSplit
split.getLocations(), split.getLength(), blockletInfos, split.getVersion(),
split.getDeleteDeltaFiles());
blockInfo.setDetailInfo(split.getDetailInfo());
+ blockInfo.setDataMapWriterPath(split.dataMapWritePath);
tableBlockInfoList.add(blockInfo);
} catch (IOException e) {
throw new RuntimeException("fail to get location of split: " + split, e);
@@ -221,6 +225,10 @@ public class CarbonInputSplit extends FileSplit
detailInfo = new BlockletDetailInfo();
detailInfo.readFields(in);
}
+ boolean dataMapWriterPathExists = in.readBoolean();
+ if (dataMapWriterPathExists) {
+ dataMapWritePath = in.readUTF();
+ }
}
@Override public void write(DataOutput out) throws IOException {
@@ -242,6 +250,10 @@ public class CarbonInputSplit extends FileSplit
if (detailInfo != null) {
detailInfo.write(out);
}
+ out.writeBoolean(dataMapWritePath != null);
+ if (dataMapWritePath != null) {
+ out.writeUTF(dataMapWritePath);
+ }
}
public List<String> getInvalidSegments() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 8bf779e..97a86ea 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -35,6 +35,7 @@ import java.util.Set;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
+import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -66,6 +67,7 @@ import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
import org.apache.carbondata.core.statusmanager.FileFormat;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
+import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeConverter;
@@ -687,16 +689,17 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// get tokens for all the required FileSystem for table path
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
new Path[] { new Path(absoluteTableIdentifier.getTablePath()) }, job.getConfiguration());
-
- TableDataMap blockletMap = DataMapStoreManager.getInstance()
- .getDataMap(absoluteTableIdentifier, BlockletDataMap.NAME,
- BlockletDataMapFactory.class.getName());
+ boolean distributedCG = Boolean.parseBoolean(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT));
+ TableDataMap blockletMap =
+ DataMapStoreManager.getInstance().chooseDataMap(absoluteTableIdentifier);
DataMapJob dataMapJob = getDataMapJob(job.getConfiguration());
List<ExtendedBlocklet> prunedBlocklets;
- if (dataMapJob != null) {
+ if (distributedCG || blockletMap.getDataMapFactory().getDataMapType() == DataMapType.FG) {
DistributableDataMapFormat datamapDstr =
- new DistributableDataMapFormat(absoluteTableIdentifier, BlockletDataMap.NAME,
- segmentIds, BlockletDataMapFactory.class.getName());
+ new DistributableDataMapFormat(absoluteTableIdentifier, blockletMap.getDataMapName(),
+ segmentIds, blockletMap.getDataMapFactory().getClass().getName());
prunedBlocklets = dataMapJob.execute(datamapDstr, resolver);
} else {
prunedBlocklets = blockletMap.prune(segmentIds, resolver);
@@ -752,10 +755,12 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
return null;
}
org.apache.carbondata.hadoop.CarbonInputSplit split =
- org.apache.carbondata.hadoop.CarbonInputSplit.from(blocklet.getSegmentId(),
+ org.apache.carbondata.hadoop.CarbonInputSplit.from(
+ blocklet.getSegmentId(),
new FileSplit(new Path(blocklet.getPath()), 0, blocklet.getLength(),
blocklet.getLocations()),
- ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()));
+ ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()),
+ blocklet.getDataMapWriterPath());
split.setDetailInfo(blocklet.getDetailInfo());
return split;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
new file mode 100644
index 0000000..5294de3
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datastore.FileHolder
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.Blocklet
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class CGDataMapFactory extends AbstractCoarseGrainDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapName: String = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier,
+ dataMapName: String): Unit = {
+ this.identifier = identifier
+ this.dataMapName = dataMapName
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter = {
+ new CGDataMapWriter(identifier, segmentId, dataWritePath, dataMapName)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segmentId: String): java.util.List[AbstractCoarseGrainDataMap] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map {f =>
+ val dataMap: AbstractCoarseGrainDataMap = new CGDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+
+ /**
+ * Get datamaps for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: AbstractCoarseGrainDataMap = new CGDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event): Unit = {
+ ???
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segmentId: String): java.util.List[DataMapDistributable] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d:DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segmentId: String): Unit = {
+
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(Seq("name").toList.asJava, new ArrayBuffer[ExpressionType]().toList.asJava)
+ }
+}
+
+class CGDataMap extends AbstractCoarseGrainDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))] = _
+ var fileHolder: FileHolder = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ fileHolder = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = fileHolder.readInt(filePath, size-4)
+ val bytes = fileHolder.readByteArray(filePath, size-footerLen-4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject().asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties): java.util.List[Blocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f=>
+ new Blocklet(f._1, f._2+"")
+ }.asJava
+ }
+
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) <= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) >= 0
+ }
+ tuples
+ }
+
+ private def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear() = {
+ ???
+ }
+}
+
+class CGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segmentId: String,
+ dataWritePath: String,
+ dataMapName: String)
+ extends AbstractDataMapWriter(identifier, segmentId, dataWritePath) {
+
+ var currentBlockId: String = null
+ val cgwritepath = dataWritePath + "/" +
+ dataMapName + System.nanoTime() + ".datamap"
+ lazy val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(cgwritepath, FileFactory.getFileType(cgwritepath))
+ val blockletList = new ArrayBuffer[Array[Byte]]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]()
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ maxMin +=
+ ((currentBlockId+"", blockletId, (sorted.last, sorted.head)))
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[Array[Byte]]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += newBytes
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ blockletList += sorted.head
+ blockletList += sorted.last
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(cgwritepath)
+ }
+
+
+}
+
+class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test cg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ sql(
+ """
+ | CREATE TABLE datamap_test_cg(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test_cg")
+ // register datamap writer
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ table.getAbsoluteTableIdentifier,
+ classOf[CGDataMapFactory].getName, "cgdatamap")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_cg OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test_cg where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index 94fbff4..62b6fc5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -18,24 +18,32 @@
package org.apache.carbondata.spark.testsuite.datamap
import java.util
+
import scala.collection.JavaConverters._
-import org.apache.spark.sql.{DataFrame, SaveMode}
+
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.{DataFrame, SaveMode}
import org.scalatest.BeforeAndAfterAll
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.dev.{DataMap, DataMapFactory, DataMapWriter}
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.schema.FilterType
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
+import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.Event
-class C2DataMapFactory() extends DataMapFactory {
+class C2DataMapFactory() extends AbstractCoarseGrainDataMapFactory {
+
+ var identifier: AbsoluteTableIdentifier = _
override def init(identifier: AbsoluteTableIdentifier,
- dataMapName: String): Unit = {}
+ dataMapName: String): Unit = {
+ this.identifier = identifier
+ }
override def fireEvent(event: Event): Unit = ???
@@ -43,13 +51,14 @@ class C2DataMapFactory() extends DataMapFactory {
override def clear(): Unit = {}
- override def getDataMaps(distributable: DataMapDistributable): java.util.List[DataMap] = ???
+ override def getDataMaps(distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = ???
- override def getDataMaps(segmentId: String): util.List[DataMap] = ???
+ override def getDataMaps(segmentId: String): util.List[AbstractCoarseGrainDataMap] = ???
- override def createWriter(segmentId: String): DataMapWriter = DataMapWriterSuite.dataMapWriterC2Mock
+ override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter =
+ DataMapWriterSuite.dataMapWriterC2Mock(identifier, segmentId, dataWritePath)
- override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, FilterType.EQUALTO)
+ override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, List(ExpressionType.EQUALS).asJava)
/**
* Get all distributable objects of a segmentid
@@ -59,6 +68,7 @@ class C2DataMapFactory() extends DataMapFactory {
override def toDistributable(segmentId: String): util.List[DataMapDistributable] = {
???
}
+
}
class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
@@ -156,9 +166,12 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
}
object DataMapWriterSuite {
+
var callbackSeq: Seq[String] = Seq[String]()
- val dataMapWriterC2Mock = new DataMapWriter {
+ def dataMapWriterC2Mock(identifier: AbsoluteTableIdentifier, segmentId: String,
+ dataWritePath: String) =
+ new AbstractDataMapWriter(identifier, segmentId, dataWritePath) {
override def onPageAdded(
blockletId: Int,
@@ -183,9 +196,21 @@ object DataMapWriterSuite {
callbackSeq :+= s"blocklet start $blockletId"
}
- override def onBlockStart(blockId: String, blockPath: String): Unit = {
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String) = {
callbackSeq :+= s"block start $blockId"
}
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish() = {
+
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
new file mode 100644
index 0000000..109364e
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.dev.fgdatamap.{AbstractFineGrainDataMap, AbstractFineGrainDataMapFactory}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datastore.FileHolder
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapName: String = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier,
+ dataMapName: String): Unit = {
+ this.identifier = identifier
+ this.dataMapName = dataMapName
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter = {
+ new FGDataMapWriter(identifier, segmentId, dataWritePath, dataMapName)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segmentId: String): java.util.List[AbstractFineGrainDataMap] = {
+ val file = FileFactory
+ .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val dataMap: AbstractFineGrainDataMap = new FGDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+ /**
+ * Get datamap for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[AbstractFineGrainDataMap]= {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: AbstractFineGrainDataMap = new FGDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segmentId: String): java.util.List[DataMapDistributable] = {
+ val file = FileFactory
+ .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d: DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event):Unit = {
+ ???
+ }
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segmentId: String): Unit = {
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(Seq("name").toList.asJava, new ArrayBuffer[ExpressionType]().toList.asJava)
+ }
+}
+
+class FGDataMap extends AbstractFineGrainDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)] = _
+ var fileHolder: FileHolder = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ fileHolder = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = fileHolder.readInt(filePath, size - 4)
+ val bytes = fileHolder.readByteArray(filePath, size - footerLen - 4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties): java.util.List[FineGrainBlocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f =>
+ readAndFindData(f, value(0).getBytes())
+ }.filter(_.isDefined).map(_.get).asJava
+ }
+
+ private def readAndFindData(meta: (String, Int, (Array[Byte], Array[Byte]), Long, Int),
+ value: Array[Byte]): Option[FineGrainBlocklet] = {
+ val bytes = fileHolder.readByteArray(filePath, meta._4, meta._5)
+ val outputStream = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(outputStream)
+ val blockletsData = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]]
+
+ import scala.collection.Searching._
+ val searching = blockletsData
+ .search[(Array[Byte], Seq[Seq[Int]], Seq[Int])]((value, Seq(Seq(0)), Seq(0)))(new Ordering[
+ (Array[Byte], Seq[Seq[Int]], Seq[Int])] {
+ override def compare(x: (Array[Byte], Seq[Seq[Int]], Seq[Int]),
+ y: (Array[Byte], Seq[Seq[Int]], Seq[Int])) = {
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(x._1, y._1)
+ }
+ })
+ if (searching.insertionPoint >= 0) {
+ val f = blockletsData(searching.insertionPoint)
+ val pages = f._3.zipWithIndex.map { p =>
+ val pg = new FineGrainBlocklet.Page
+ pg.setPageId(p._1)
+ pg.setRowId(f._2(p._2).toArray)
+ pg
+ }
+ pages
+ Some(new FineGrainBlocklet(meta._1, meta._2.toString, pages.toList.asJava))
+ } else {
+ None
+ }
+
+ }
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) >= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) <= 0
+ }
+ tuples
+ }
+
+ def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear():Unit = {
+ ???
+ }
+}
+
+class FGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segmentId: String, dataWriterPath: String, dataMapName: String)
+ extends AbstractDataMapWriter(identifier, segmentId, dataWriterPath) {
+
+ var currentBlockId: String = null
+ val fgwritepath = dataWriterPath + "/" + System.nanoTime() + ".datamap"
+ val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(fgwritepath, FileFactory.getFileType(fgwritepath))
+ val blockletList = new ArrayBuffer[(Array[Byte], Seq[Int], Seq[Int])]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]()
+ var position: Long = 0
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Seq[Int]], Seq[Int]) = null
+ var addedLast: Boolean = false
+ val blockletListUpdated = new ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]()
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3 ++ f._3)
+ addedLast = false
+ } else {
+ blockletListUpdated += oldValue
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletListUpdated += oldValue
+ }
+
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(blockletListUpdated)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ maxMin +=
+ ((currentBlockId + "", blockletId, (blockletListUpdated.head._1, blockletListUpdated.last
+ ._1), position, bytes.length))
+ position += bytes.length
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[(Array[Byte], Int)]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += ((newBytes, i))
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Int], Seq[Int]) = null
+ var addedLast: Boolean = false
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3)
+ addedLast = false
+ } else {
+ blockletList += oldValue
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletList += oldValue
+ }
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(fgwritepath)
+ }
+
+
+}
+
+class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test fg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ table.getAbsoluteTableIdentifier,
+ classOf[FGDataMapFactory].getName, "fgdatamap")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index b24562c..501522b 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -353,11 +353,7 @@ class CarbonScanRDD(
identifier.appendWithLocalPrefix(identifier.getTablePath))
CarbonTableInputFormat.setFilterPredicates(conf, filterExpression)
CarbonTableInputFormat.setColumnProjection(conf, columnProjection)
- if (CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
- CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT).toBoolean) {
- CarbonTableInputFormat.setDataMapJob(conf, new SparkDataMapJob)
- }
+ CarbonTableInputFormat.setDataMapJob(conf, new SparkDataMapJob)
val dbName = identifier.getCarbonTableIdentifier.getDatabaseName.toLowerCase
val tbName = identifier.getCarbonTableIdentifier.getTableName.toLowerCase
val segmentNumbersFromProperty = CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 8e350d9..31a6701 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.processing.datamap;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -28,8 +29,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
import org.apache.carbondata.core.datamap.TableDataMap;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.processing.store.TablePage;
@@ -43,25 +44,26 @@ public class DataMapWriterListener {
DataMapWriterListener.class.getCanonicalName());
// list indexed column name -> list of data map writer
- private Map<List<String>, List<DataMapWriter>> registry = new ConcurrentHashMap<>();
+ private Map<List<String>, List<AbstractDataMapWriter>> registry = new ConcurrentHashMap<>();
/**
* register all datamap writer for specified table and segment
*/
- public void registerAllWriter(AbsoluteTableIdentifier identifier, String segmentId) {
+ public void registerAllWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String dataWritePath) {
List<TableDataMap> tableDataMaps = DataMapStoreManager.getInstance().getAllDataMap(identifier);
if (tableDataMaps != null) {
for (TableDataMap tableDataMap : tableDataMaps) {
DataMapFactory factory = tableDataMap.getDataMapFactory();
- register(factory, segmentId);
+ register(factory, segmentId, dataWritePath);
}
}
}
/**
- * Register a DataMapWriter
+ * Register a AbstractDataMapWriter
*/
- private void register(DataMapFactory factory, String segmentId) {
+ private void register(DataMapFactory factory, String segmentId, String dataWritePath) {
assert (factory != null);
assert (segmentId != null);
DataMapMeta meta = factory.getMeta();
@@ -70,8 +72,8 @@ public class DataMapWriterListener {
return;
}
List<String> columns = factory.getMeta().getIndexedColumns();
- List<DataMapWriter> writers = registry.get(columns);
- DataMapWriter writer = factory.createWriter(segmentId);
+ List<AbstractDataMapWriter> writers = registry.get(columns);
+ AbstractDataMapWriter writer = factory.createWriter(segmentId, dataWritePath);
if (writers != null) {
writers.add(writer);
} else {
@@ -79,36 +81,36 @@ public class DataMapWriterListener {
writers.add(writer);
registry.put(columns, writers);
}
- LOG.info("DataMapWriter " + writer + " added");
+ LOG.info("AbstractDataMapWriter " + writer + " added");
}
public void onBlockStart(String blockId, String blockPath) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
- writer.onBlockStart(blockId, blockPath);
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
+ writer.onBlockStart(blockId);
}
}
}
public void onBlockEnd(String blockId) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
writer.onBlockEnd(blockId);
}
}
}
public void onBlockletStart(int blockletId) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
writer.onBlockletStart(blockletId);
}
}
}
public void onBlockletEnd(int blockletId) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
writer.onBlockletEnd(blockletId);
}
}
@@ -121,18 +123,29 @@ public class DataMapWriterListener {
* @param tablePage page data
*/
public void onPageAdded(int blockletId, int pageId, TablePage tablePage) {
- Set<Map.Entry<List<String>, List<DataMapWriter>>> entries = registry.entrySet();
- for (Map.Entry<List<String>, List<DataMapWriter>> entry : entries) {
+ Set<Map.Entry<List<String>, List<AbstractDataMapWriter>>> entries = registry.entrySet();
+ for (Map.Entry<List<String>, List<AbstractDataMapWriter>> entry : entries) {
List<String> indexedColumns = entry.getKey();
ColumnPage[] pages = new ColumnPage[indexedColumns.size()];
for (int i = 0; i < indexedColumns.size(); i++) {
pages[i] = tablePage.getColumnPage(indexedColumns.get(i));
}
- List<DataMapWriter> writers = entry.getValue();
- for (DataMapWriter writer : writers) {
+ List<AbstractDataMapWriter> writers = entry.getValue();
+ for (AbstractDataMapWriter writer : writers) {
writer.onPageAdded(blockletId, pageId, pages);
}
}
}
+ /**
+ * Finish all datamap writers
+ */
+ public void finish() throws IOException {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
+ writer.finish();
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 4f9458c..cc38b16 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.TableSpec;
@@ -259,7 +260,8 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.sortScope = CarbonDataProcessorUtil.getSortScope(configuration);
DataMapWriterListener listener = new DataMapWriterListener();
- listener.registerAllWriter(configuration.getTableIdentifier(), configuration.getSegmentId());
+ listener.registerAllWriter(configuration.getTableIdentifier(), configuration.getSegmentId(),
+ storeLocation[new Random().nextInt(storeLocation.length)]);
carbonFactDataHandlerModel.dataMapWriterlistener = listener;
return carbonFactDataHandlerModel;
@@ -322,6 +324,12 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.tableSpec = new TableSpec(
segmentProperties.getDimensions(),
segmentProperties.getMeasures());
+ DataMapWriterListener listener = new DataMapWriterListener();
+ listener.registerAllWriter(
+ loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier(),
+ loadModel.getSegmentId(),
+ tempStoreLocation[new Random().nextInt(tempStoreLocation.length)]);
+ carbonFactDataHandlerModel.dataMapWriterlistener = listener;
return carbonFactDataHandlerModel;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/36c34cd0/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index b00ec70..04e304f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.processing.store.writer;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
@@ -41,14 +39,11 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonMergerUtil;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.util.CarbonProperties;
@@ -62,7 +57,6 @@ import org.apache.carbondata.format.IndexHeader;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
import org.apache.commons.lang3.ArrayUtils;
-import org.apache.hadoop.io.IOUtils;
public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
@@ -70,12 +64,6 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
LogServiceFactory.getLogService(AbstractFactDataWriter.class.getName());
/**
- * dfs.bytes-per-checksum
- * HDFS checksum length, block size for a file should be exactly divisible
- * by this value
- */
- private static final int HDFS_CHECKSUM_LENGTH = 512;
- /**
* file channel
*/
protected FileChannel fileChannel;
@@ -207,35 +195,6 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
}
/**
- * This method will return max of block size and file size
- *
- * @param blockSize
- * @param fileSize
- * @return
- */
- private static long getMaxOfBlockAndFileSize(long blockSize, long fileSize) {
- long maxSize = blockSize;
- if (fileSize > blockSize) {
- maxSize = fileSize;
- }
- // block size should be exactly divisible by 512 which is maintained by HDFS as bytes
- // per checksum, dfs.bytes-per-checksum=512 must divide block size
- long remainder = maxSize % HDFS_CHECKSUM_LENGTH;
- if (remainder > 0) {
- maxSize = maxSize + HDFS_CHECKSUM_LENGTH - remainder;
- }
- // convert to make block size more readable.
- String readableBlockSize = ByteUtil.convertByteToReadable(blockSize);
- String readableFileSize = ByteUtil.convertByteToReadable(fileSize);
- String readableMaxSize = ByteUtil.convertByteToReadable(maxSize);
- LOGGER.info(
- "The configured block size is " + readableBlockSize + ", the actual carbon file size is "
- + readableFileSize + ", choose the max value " + readableMaxSize
- + " as the block size on HDFS");
- return maxSize;
- }
-
- /**
* This method will be used to update the file channel with new file if exceeding block size
* threshold, new file will be created once existing file reached the file size limit This
* method will first check whether existing file size is exceeded the file
@@ -296,7 +255,9 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
String fileName = this.carbonDataFileTempPath.substring(0,
this.carbonDataFileTempPath.lastIndexOf('.'));
if (copyInCurrentThread) {
- copyCarbonDataFileToCarbonStorePath(fileName);
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(
+ fileName, dataWriterVo.getCarbonDataDirectoryPath(),
+ fileSizeInBytes);
} else {
executorServiceSubmitList.add(executorService.submit(new CopyThread(fileName)));
}
@@ -449,7 +410,9 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
}
writer.close();
// copy from temp to actual store location
- copyCarbonDataFileToCarbonStorePath(fileName);
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(fileName,
+ dataWriterVo.getCarbonDataDirectoryPath(),
+ fileSizeInBytes);
}
/**
@@ -459,20 +422,16 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
* @throws CarbonDataWriterException
*/
protected void closeExecutorService() throws CarbonDataWriterException {
- executorService.shutdown();
try {
+ listener.finish();
+ executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- throw new CarbonDataWriterException(e.getMessage());
- }
- for (int i = 0; i < executorServiceSubmitList.size(); i++) {
- try {
+ for (int i = 0; i < executorServiceSubmitList.size(); i++) {
executorServiceSubmitList.get(i).get();
- } catch (InterruptedException e) {
- throw new CarbonDataWriterException(e.getMessage());
- } catch (ExecutionException e) {
- throw new CarbonDataWriterException(e.getMessage());
}
+ } catch (InterruptedException | ExecutionException | IOException e) {
+ LOGGER.error(e, "Error while finishing writer");
+ throw new CarbonDataWriterException(e.getMessage());
}
}
@@ -490,62 +449,6 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
}
}
- /**
- * This method will copy the given file to carbon store location
- *
- * @param localFileName local file name with full path
- * @throws CarbonDataWriterException
- */
- protected void copyCarbonDataFileToCarbonStorePath(String localFileName)
- throws CarbonDataWriterException {
- long copyStartTime = System.currentTimeMillis();
- LOGGER.info("Copying " + localFileName + " --> " + dataWriterVo.getCarbonDataDirectoryPath());
- try {
- CarbonFile localCarbonFile =
- FileFactory.getCarbonFile(localFileName, FileFactory.getFileType(localFileName));
- String carbonFilePath = dataWriterVo.getCarbonDataDirectoryPath() + localFileName
- .substring(localFileName.lastIndexOf(File.separator));
- copyLocalFileToCarbonStore(carbonFilePath, localFileName,
- CarbonCommonConstants.BYTEBUFFER_SIZE,
- getMaxOfBlockAndFileSize(fileSizeInBytes, localCarbonFile.getSize()));
- } catch (IOException e) {
- throw new CarbonDataWriterException(
- "Problem while copying file from local store to carbon store", e);
- }
- LOGGER.info(
- "Total copy time (ms) to copy file " + localFileName + " is " + (System.currentTimeMillis()
- - copyStartTime));
- }
-
- /**
- * This method will read the local carbon data file and write to carbon data file in HDFS
- *
- * @param carbonStoreFilePath
- * @param localFilePath
- * @param bufferSize
- * @param blockSize
- * @throws IOException
- */
- private void copyLocalFileToCarbonStore(String carbonStoreFilePath, String localFilePath,
- int bufferSize, long blockSize) throws IOException {
- DataOutputStream dataOutputStream = null;
- DataInputStream dataInputStream = null;
- try {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize
- + " (bytes");
- }
- dataOutputStream = FileFactory
- .getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
- bufferSize, blockSize);
- dataInputStream = FileFactory
- .getDataInputStream(localFilePath, FileFactory.getFileType(localFilePath), bufferSize);
- IOUtils.copyBytes(dataInputStream, dataOutputStream, bufferSize);
- } finally {
- CarbonUtil.closeStream(dataInputStream);
- CarbonUtil.closeStream(dataOutputStream);
- }
- }
/**
* This method will copy the carbon data file from local store location to
@@ -570,7 +473,10 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
* @throws Exception if unable to compute a result
*/
@Override public Void call() throws Exception {
- copyCarbonDataFileToCarbonStorePath(fileName);
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(
+ fileName,
+ dataWriterVo.getCarbonDataDirectoryPath(),
+ fileSizeInBytes);
return null;
}
[04/28] carbondata git commit: [CARBONDATA-1608]Support Column
Comment for Create Table This closes #1432
Posted by ja...@apache.org.
[CARBONDATA-1608]Support Column Comment for Create Table This closes #1432
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/808a334f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/808a334f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/808a334f
Branch: refs/heads/fgdatamap
Commit: 808a334f03ea00856e7c6556418835e6d29c1e50
Parents: 17892b1 9c9521b
Author: kumarvishal <ku...@gmail.com>
Authored: Thu Nov 16 20:42:10 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Thu Nov 16 20:42:10 2017 +0530
----------------------------------------------------------------------
.../TestCreateTableWithColumnComment.scala | 54 ++++++++++++++++++++
.../CarbonDescribeFormattedCommand.scala | 19 ++++---
.../sql/parser/CarbonSpark2SqlParser.scala | 14 +++--
.../BooleanDataTypesInsertTest.scala | 40 +++++++++++++++
4 files changed, 115 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
[13/28] carbondata git commit: [HOTFIX] change to use store path in
property in testcase
Posted by ja...@apache.org.
[HOTFIX] change to use store path in property in testcase
This closes #1522
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c3e326e0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c3e326e0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c3e326e0
Branch: refs/heads/fgdatamap
Commit: c3e326e021e4d530b0d30a82c55a09f40a34c2fe
Parents: 0f46ef0
Author: Jacky Li <ja...@huawei.com>
Authored: Fri Nov 17 15:45:26 2017 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Fri Nov 17 15:50:26 2017 +0800
----------------------------------------------------------------------
.../cluster/sdv/generated/CarbonV1toV3CompatabilityTestCase.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e326e0/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CarbonV1toV3CompatabilityTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CarbonV1toV3CompatabilityTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CarbonV1toV3CompatabilityTestCase.scala
index f49e475..93971b0 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CarbonV1toV3CompatabilityTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CarbonV1toV3CompatabilityTestCase.scala
@@ -47,7 +47,7 @@ class CarbonV1toV3CompatabilityTestCase extends QueryTest with BeforeAndAfterAll
.appName("CarbonV1toV3CompatabilityTestCase")
.config("spark.driver.host", "localhost")
.getOrCreateCarbonSession(storeLocation, metaLocation).asInstanceOf[CarbonSession]
- println("store path from env : " + CarbonEnv.getInstance(localspark).storePath)
+ println("store path : " + CarbonProperties.getStorePath)
localspark.sparkContext.setLogLevel("WARN")
localspark.sessionState.asInstanceOf[CarbonSessionState].metadataHive
.runSqlHive(
[24/28] carbondata git commit: [CARBONDATA-1751] Modify sys.err to
AnalysisException when uses run related operation except IUD,
compaction and alter
Posted by ja...@apache.org.
[CARBONDATA-1751] Modify sys.err to AnalysisException when uses run related operation except IUD,compaction and alter
carbon printout improper error message, for example, it printout system error when users run create table with the same column name, but it should printout related exception information
So we modify sys.error method to AnalysisException when uses run related operation except IUD,compaction and alter
This closes #1515
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2a12938b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2a12938b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2a12938b
Branch: refs/heads/fgdatamap
Commit: 2a12938b545cca5e3c09396dd68393ea615038fa
Parents: 1b8d348
Author: xubo245 <60...@qq.com>
Authored: Fri Nov 17 10:48:31 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Nov 18 16:24:23 2017 +0800
----------------------------------------------------------------------
.../command/CarbonTableSchemaCommonSuite.scala | 72 ++++++++++++++++++++
.../org/apache/carbondata/api/CarbonStore.scala | 4 +-
.../carbondata/spark/util/CommonUtil.scala | 17 ++---
.../spark/util/DataTypeConverterUtil.scala | 4 +-
.../catalyst/AbstractCarbonSparkSQLParser.scala | 3 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 4 +-
.../command/carbonTableSchemaCommon.scala | 10 +--
.../apache/spark/sql/util/CarbonException.scala | 24 +++++++
.../sql/parser/CarbonSpark2SqlParser.scala | 2 +-
.../apache/spark/util/CarbonCommandSuite.scala | 2 +-
10 files changed, 124 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark-common-test/src/test/scala/org/apache/spark/sql/execution/command/CarbonTableSchemaCommonSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/spark/sql/execution/command/CarbonTableSchemaCommonSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/spark/sql/execution/command/CarbonTableSchemaCommonSuite.scala
new file mode 100644
index 0000000..67dfa8f
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/spark/sql/execution/command/CarbonTableSchemaCommonSuite.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.test.util.QueryTest
+import org.junit.Assert
+import org.scalatest.BeforeAndAfterAll
+
+class CarbonTableSchemaCommonSuite extends QueryTest with BeforeAndAfterAll {
+
+ test("Creating table: Duplicate dimensions found with name, it should throw AnalysisException") {
+ sql("DROP TABLE IF EXISTS carbon_table")
+ try {
+ sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | BB INT, bb char(10)
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+ Assert.assertTrue(false)
+ } catch {
+ case _: AnalysisException => Assert.assertTrue(true)
+ case _: Exception => Assert.assertTrue(false)
+ } finally {
+ sql("DROP TABLE IF EXISTS carbon_table")
+ }
+ }
+
+ test("Altering table: Duplicate column found with name, it should throw RuntimeException") {
+ sql("DROP TABLE IF EXISTS carbon_table")
+ sql(
+ s"""
+ | CREATE TABLE if not exists carbon_table(
+ | BB INT, cc char(10)
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
+
+ try {
+ sql(
+ s"""
+ | alter TABLE carbon_table add columns(
+ | bb char(10)
+ )
+ """.stripMargin)
+ Assert.assertTrue(false)
+ } catch {
+ case _: RuntimeException => Assert.assertTrue(true)
+ case _: Exception => Assert.assertTrue(false)
+ } finally {
+ sql("DROP TABLE IF EXISTS carbon_table")
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index 6c2490e..73325a6 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -24,6 +24,7 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.expressions.{Cast, Literal}
import org.apache.spark.sql.types.TimestampType
+import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -57,7 +58,8 @@ object CarbonStore {
val lim = Integer.parseInt(limitLoads)
loadMetadataDetailsSortedArray = loadMetadataDetailsSortedArray.slice(0, lim)
} catch {
- case _: NumberFormatException => sys.error(s" Entered limit is not a valid Number")
+ case _: NumberFormatException =>
+ CarbonException.analysisException("Entered limit is not a valid Number")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index a922a07..bed57bd 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.{Row, RowFactory}
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
import org.apache.spark.sql.execution.command.{ColumnProperty, Field, PartitionerField}
import org.apache.spark.sql.types.{MetadataBuilder, StringType}
+import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.FileUtils
import org.apache.carbondata.common.logging.LogServiceFactory
@@ -662,19 +663,19 @@ object CommonUtil {
val maxColumnsInt = getMaxColumnValue(maxColumns)
if (maxColumnsInt != null) {
if (columnCountInSchema >= maxColumnsInt) {
- sys.error(s"csv headers should be less than the max columns: $maxColumnsInt")
+ CarbonException.analysisException(
+ s"csv headers should be less than the max columns: $maxColumnsInt")
} else if (maxColumnsInt > CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING) {
- sys.error(s"max columns cannot be greater than the threshold value: ${
- CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING
- }")
+ CarbonException.analysisException(
+ s"max columns cannot be greater than the threshold value: " +
+ s"${CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING}")
} else {
maxNumberOfColumnsForParsing = maxColumnsInt
}
} else if (columnCountInSchema >= CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING) {
- sys.error(s"csv header columns should be less than max threashold: ${
- CSVInputFormat
- .THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING
- }")
+ CarbonException.analysisException(
+ s"csv header columns should be less than max threashold: " +
+ s"${CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING}")
} else if (columnCountInSchema >= CSVInputFormat.DEFAULT_MAX_NUMBER_OF_COLUMNS_FOR_PARSING) {
maxNumberOfColumnsForParsing = columnCountInSchema + 1
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 0969fdf..6673e18 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -17,6 +17,8 @@
package org.apache.carbondata.spark.util
+import org.apache.spark.sql.util.CarbonException
+
import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
import org.apache.carbondata.format.{DataType => ThriftDataType}
@@ -75,7 +77,7 @@ object DataTypeConverterUtil {
} else if (others != null && others.startsWith("varchar")) {
DataTypes.STRING
} else {
- sys.error(s"Unsupported data type: $dataType")
+ CarbonException.analysisException(s"Unsupported data type: $dataType")
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/AbstractCarbonSparkSQLParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/AbstractCarbonSparkSQLParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/AbstractCarbonSparkSQLParser.scala
index fba7976..2dc2d4e 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/AbstractCarbonSparkSQLParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/AbstractCarbonSparkSQLParser.scala
@@ -24,6 +24,7 @@ import scala.util.parsing.combinator.PackratParsers
import scala.util.parsing.input.CharArrayReader.EofCh
import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.util.CarbonException
private[sql] abstract class AbstractCarbonSparkSQLParser
extends StandardTokenParsers with PackratParsers {
@@ -33,7 +34,7 @@ private[sql] abstract class AbstractCarbonSparkSQLParser
initLexical
phrase(start)(new lexical.Scanner(input)) match {
case Success(plan, _) => plan
- case failureOrError => sys.error(failureOrError.toString)
+ case failureOrError => CarbonException.analysisException(failureOrError.toString)
}
}
/* One time initialization of lexical.This avoid reinitialization of lexical in parse method */
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 0e77e8c..b4cde9b 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -28,8 +28,10 @@ import scala.util.matching.Regex
import org.apache.hadoop.hive.ql.lib.Node
import org.apache.hadoop.hive.ql.parse._
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.trees.CurrentOrigin
import org.apache.spark.sql.execution.command._
+import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.PartitionUtils
import org.apache.carbondata.common.constants.LoggerAction
@@ -781,7 +783,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
if (remainingNodes.nonEmpty) {
- sys.error(
+ CarbonException.analysisException(
s"""Unhandled clauses:
|You are likely trying to use an unsupported carbon feature."""".stripMargin)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 9c927aa..d64544d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -26,6 +26,7 @@ import scala.collection.mutable.Map
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -530,7 +531,7 @@ class TableNewProcessor(cm: TableModel) {
s"Validation failed for Create/Alter Table Operation " +
s"for ${ cm.databaseName }.${ cm.tableName }" +
s"Duplicate column found with name: $name")
- sys.error(s"Duplicate dimensions found with name: $name")
+ CarbonException.analysisException(s"Duplicate dimensions found with name: $name")
}
}
@@ -600,12 +601,12 @@ class TableNewProcessor(cm: TableModel) {
} else {
LOGGER.error(s"Bucket field must be dimension column and " +
s"should not be measure or complex column: ${colSchema.getColumnName}")
- sys.error(s"Bucket field must be dimension column and " +
+ CarbonException.analysisException(s"Bucket field must be dimension column and " +
s"should not be measure or complex column: ${colSchema.getColumnName}")
}
case _ =>
LOGGER.error(s"Bucket field is not present in table columns")
- sys.error(s"Bucket field is not present in table columns")
+ CarbonException.analysisException(s"Bucket field is not present in table columns")
}
}
tableSchema.setBucketingInfo(
@@ -653,7 +654,8 @@ class TableNewProcessor(cm: TableModel) {
})
if (!found) {
- sys.error(s"column $colForGrouping is not present in Field list")
+ CarbonException.analysisException(
+ s"column $colForGrouping is not present in Field list")
}
})
})
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark-common/src/main/scala/org/apache/spark/sql/util/CarbonException.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/util/CarbonException.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/util/CarbonException.scala
new file mode 100644
index 0000000..9fd7099
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/util/CarbonException.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.util
+
+ import org.apache.spark.sql.AnalysisException
+
+ object CarbonException {
+ def analysisException(message: String): Nothing = throw new AnalysisException(message)
+ }
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index 7a637f7..c8ea4ac 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -479,7 +479,7 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
match {
case Success(field, _) => field.asInstanceOf[Field]
case failureOrError => throw new MalformedCarbonCommandException(
- s"Unsupported data type: $col.getType")
+ s"Unsupported data type: ${ col.dataType }")
}
// the data type of the decimal type will be like decimal(10,0)
// so checking the start of the string and taking the precision and scale.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a12938b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
index 9f89226..9e20cdd 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
@@ -100,7 +100,7 @@ class CarbonCommandSuite extends Spark2QueryTest with BeforeAndAfterAll {
test("delete segment by id") {
DeleteSegmentById.main(Array(s"${location}", "carbon_table", "0"))
- assert(!CarbonStore.isSegmentValid("default", "carbon_table",location, "0"))
+ assert(!CarbonStore.isSegmentValid("default", "carbon_table", location, "0"))
}
test("delete segment by date") {
[06/28] carbondata git commit: [CARBONDATA-1733] While load is in
progress, Show segments is throwing NPE This closes #1505
Posted by ja...@apache.org.
[CARBONDATA-1733] While load is in progress, Show segments is throwing NPE This closes #1505
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6551620b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6551620b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6551620b
Branch: refs/heads/fgdatamap
Commit: 6551620b2db60667aeb0fb95bd573759d2bd4636
Parents: 808a334 aff3b9e
Author: kumarvishal <ku...@gmail.com>
Authored: Thu Nov 16 20:48:01 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Thu Nov 16 20:48:01 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/statusmanager/LoadMetadataDetails.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
[19/28] carbondata git commit: [CARBONDATA-1626]add data size and
index size in table status file
Posted by ja...@apache.org.
[CARBONDATA-1626]add data size and index size in table status file
This closes #1435
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/589f126d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/589f126d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/589f126d
Branch: refs/heads/fgdatamap
Commit: 589f126dea872f54c2096c9572436bf10589b1ca
Parents: f22e614
Author: akashrn5 <ak...@gmail.com>
Authored: Wed Oct 25 15:27:37 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Nov 17 21:43:15 2017 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 26 +++
.../core/datastore/impl/FileFactory.java | 2 +-
.../core/statusmanager/LoadMetadataDetails.java | 18 ++
.../apache/carbondata/core/util/CarbonUtil.java | 152 ++++++++++++++++
.../core/util/path/CarbonTablePath.java | 8 +
.../spark/rdd/CarbonDataRDDFactory.scala | 11 +-
.../CarbonDescribeFormattedCommand.scala | 10 ++
.../spark/sql/GetDataSizeAndIndexSizeTest.scala | 172 +++++++++++++++++++
.../processing/merger/CarbonDataMergerUtil.java | 7 +-
9 files changed, 398 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/589f126d/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 0a7dfdd..762ef6d 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1380,6 +1380,32 @@ public final class CarbonCommonConstants {
public static final String AGGREGATIONDATAMAPSCHEMA = "AggregateDataMapHandler";
+ /*
+ * The total size of carbon data
+ */
+ public static final String CARBON_TOTAL_DATA_SIZE = "datasize";
+
+ /**
+ * The total size of carbon index
+ */
+ public static final String CARBON_TOTAL_INDEX_SIZE = "indexsize";
+
+ /**
+ * ENABLE_CALCULATE_DATA_INDEX_SIZE
+ */
+ @CarbonProperty public static final String ENABLE_CALCULATE_SIZE = "carbon.enable.calculate.size";
+
+ /**
+ * DEFAULT_ENABLE_CALCULATE_DATA_INDEX_SIZE
+ */
+ @CarbonProperty public static final String DEFAULT_ENABLE_CALCULATE_SIZE = "true";
+
+ public static final String TABLE_DATA_SIZE = "Table Data Size";
+
+ public static final String TABLE_INDEX_SIZE = "Table Index Size";
+
+ public static final String LAST_UPDATE_TIME = "Last Update Time";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/589f126d/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index 57a48ec..240253d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -541,7 +541,7 @@ public final class FileFactory {
* @param fileType
* @return updated file path without url for local
*/
- private static String getUpdatedFilePath(String filePath, FileType fileType) {
+ public static String getUpdatedFilePath(String filePath, FileType fileType) {
switch (fileType) {
case HDFS:
case ALLUXIO:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/589f126d/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index d838e2e..b282d53 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -41,6 +41,24 @@ public class LoadMetadataDetails implements Serializable {
private String partitionCount;
private String isDeleted = CarbonCommonConstants.KEYWORD_FALSE;
+ private String dataSize;
+ private String indexSize;
+
+ public String getDataSize() {
+ return dataSize;
+ }
+
+ public void setDataSize(String dataSize) {
+ this.dataSize = dataSize;
+ }
+
+ public String getIndexSize() {
+ return indexSize;
+ }
+
+ public void setIndexSize(String indexSize) {
+ this.indexSize = indexSize;
+ }
// update delta end timestamp
private String updateDeltaEndTimestamp = "";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/589f126d/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 3c177dc..9d6acb6 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -58,6 +58,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
+import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
@@ -67,6 +68,7 @@ import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
@@ -77,6 +79,9 @@ import org.apache.carbondata.core.reader.ThriftReader.TBaseCreator;
import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.PathService;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -86,8 +91,11 @@ import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.DataChunk3;
import com.google.gson.Gson;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.TBase;
@@ -2140,5 +2148,149 @@ public final class CarbonUtil {
return parentPath.toString() + CarbonCommonConstants.FILE_SEPARATOR + carbonTableIdentifier
.getTableName();
}
+
+ /*
+ * This method will add data size and index size into tablestatus for each segment
+ */
+ public static void addDataIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails,
+ String segmentId, CarbonTable carbonTable) throws IOException {
+ CarbonTablePath carbonTablePath =
+ CarbonStorePath.getCarbonTablePath((carbonTable.getAbsoluteTableIdentifier()));
+ Map<String, Long> dataIndexSize =
+ CarbonUtil.getDataSizeAndIndexSize(carbonTablePath, segmentId);
+ loadMetadataDetails
+ .setDataSize(dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE).toString());
+ loadMetadataDetails
+ .setIndexSize(dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE).toString());
+ }
+
+ /**
+ * This method will calculate the data size and index size for carbon table
+ */
+ public static Map<String, Long> calculateDataIndexSize(CarbonTable carbonTable)
+ throws IOException {
+ Map<String, Long> dataIndexSizeMap = new HashMap<String, Long>();
+ long dataSize = 0L;
+ long indexSize = 0L;
+ long lastUpdateTime = 0L;
+ boolean needUpdate = false;
+ AbsoluteTableIdentifier absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
+ CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
+ String isCalculated = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.ENABLE_CALCULATE_SIZE,
+ CarbonCommonConstants.DEFAULT_ENABLE_CALCULATE_SIZE);
+ if (isCalculated.equalsIgnoreCase("true")) {
+ SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
+ ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
+ try {
+ if (carbonLock.lockWithRetries()) {
+ LOGGER.info("Acquired lock for table for table status updation");
+ String metadataPath = carbonTable.getMetaDataFilepath();
+ LoadMetadataDetails[] loadMetadataDetails =
+ SegmentStatusManager.readLoadMetadata(metadataPath);
+
+ for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
+ SegmentStatus loadStatus = loadMetadataDetail.getSegmentStatus();
+ if (loadStatus == SegmentStatus.SUCCESS || loadStatus ==
+ SegmentStatus.LOAD_PARTIAL_SUCCESS) {
+ String dsize = loadMetadataDetail.getDataSize();
+ String isize = loadMetadataDetail.getIndexSize();
+ // If it is old segment, need to calculate data size and index size again
+ if (null == dsize || null == isize) {
+ needUpdate = true;
+ LOGGER.info("It is an old segment, need calculate data size and index size again");
+ HashMap<String, Long> map = CarbonUtil
+ .getDataSizeAndIndexSize(carbonTablePath, loadMetadataDetail.getLoadName());
+ dsize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE));
+ isize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE));
+ loadMetadataDetail.setDataSize(dsize);
+ loadMetadataDetail.setIndexSize(isize);
+ }
+ dataSize += Long.parseLong(dsize);
+ indexSize += Long.parseLong(isize);
+ }
+ }
+ // If it contains old segment, write new load details
+ if (needUpdate) {
+ SegmentStatusManager.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(),
+ loadMetadataDetails);
+ }
+ String tableStatusPath = carbonTablePath.getTableStatusFilePath();
+ if (FileFactory.isFileExist(tableStatusPath, FileFactory.getFileType(tableStatusPath))) {
+ lastUpdateTime =
+ FileFactory.getCarbonFile(tableStatusPath, FileFactory.getFileType(tableStatusPath))
+ .getLastModifiedTime();
+ }
+ dataIndexSizeMap
+ .put(String.valueOf(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE), dataSize);
+ dataIndexSizeMap
+ .put(String.valueOf(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE), indexSize);
+ dataIndexSizeMap
+ .put(String.valueOf(CarbonCommonConstants.LAST_UPDATE_TIME), lastUpdateTime);
+ } else {
+ LOGGER.error("Not able to acquire the lock for Table status updation for table");
+ }
+ } finally {
+ if (carbonLock.unlock()) {
+ LOGGER.info("Table unlocked successfully after table status updation");
+ } else {
+ LOGGER.error("Unable to unlock Table lock for table during table status updation");
+ }
+ }
+ }
+ return dataIndexSizeMap;
+ }
+
+ // Get the total size of carbon data and the total size of carbon index
+ public static HashMap<String, Long> getDataSizeAndIndexSize(CarbonTablePath carbonTablePath,
+ String segmentId) throws IOException {
+ long carbonDataSize = 0L;
+ long carbonIndexSize = 0L;
+ HashMap<String, Long> dataAndIndexSize = new HashMap<String, Long>();
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ FileFactory.FileType fileType = FileFactory.getFileType(segmentPath);
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ case S3:
+ Path path = new Path(segmentPath);
+ FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
+ FileStatus[] fileStatuses = fs.listStatus(path);
+ if (null != fileStatuses) {
+ for (FileStatus dataAndIndexStatus : fileStatuses) {
+ String pathName = dataAndIndexStatus.getPath().getName();
+ if (pathName.endsWith(CarbonTablePath.getCarbonIndexExtension()) || pathName
+ .endsWith(CarbonTablePath.getCarbonMergeIndexExtension())) {
+ carbonIndexSize += dataAndIndexStatus.getLen();
+ } else if (pathName.endsWith(CarbonTablePath.getCarbonDataExtension())) {
+ carbonDataSize += dataAndIndexStatus.getLen();
+ }
+ }
+ }
+ break;
+ case LOCAL:
+ default:
+ segmentPath = FileFactory.getUpdatedFilePath(segmentPath, fileType);
+ File file = new File(segmentPath);
+ File[] segmentFiles = file.listFiles();
+ if (null != segmentFiles) {
+ for (File dataAndIndexFile : segmentFiles) {
+ if (dataAndIndexFile.getCanonicalPath()
+ .endsWith(CarbonTablePath.getCarbonIndexExtension()) || dataAndIndexFile
+ .getCanonicalPath().endsWith(CarbonTablePath.getCarbonMergeIndexExtension())) {
+ carbonIndexSize += FileUtils.sizeOf(dataAndIndexFile);
+ } else if (dataAndIndexFile.getCanonicalPath()
+ .endsWith(CarbonTablePath.getCarbonDataExtension())) {
+ carbonDataSize += FileUtils.sizeOf(dataAndIndexFile);
+ }
+ }
+ }
+ }
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE, carbonDataSize);
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, carbonIndexSize);
+ return dataAndIndexSize;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/589f126d/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index d363ac3..376a71f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -681,6 +681,14 @@ public class CarbonTablePath extends Path {
}
/**
+ *
+ * @return carbon index merge file extension
+ */
+ public static String getCarbonMergeIndexExtension() {
+ return MERGE_INDEX_FILE_EXT;
+ }
+
+ /**
* This method will remove strings in path and return short block id
*
* @param blockId
http://git-wip-us.apache.org/repos/asf/carbondata/blob/589f126d/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index e32c407..7dad243 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -55,11 +55,8 @@ import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.events.{LoadTablePostExecutionEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.FailureCauses
-import org.apache.carbondata.processing.loading.csvinput.BlockDetails
-import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
-import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable
-import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
-import org.apache.carbondata.processing.loading.exception.NoRetryException
+import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat, StringArrayWritable}
+import org.apache.carbondata.processing.loading.exception.{CarbonDataLoadingException, NoRetryException}
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.loading.sort.SortScopeOptions
import org.apache.carbondata.processing.merger.{CarbonCompactionUtil, CarbonDataMergerUtil, CompactionType}
@@ -292,6 +289,7 @@ object CarbonDataRDDFactory {
var executorMessage: String = ""
val isSortTable = carbonTable.getNumberOfSortColumns > 0
val sortScope = CarbonDataProcessorUtil.getSortScope(carbonLoadModel.getSortScope)
+
try {
if (updateModel.isDefined) {
res = loadDataFrameForUpdate(
@@ -748,6 +746,7 @@ object CarbonDataRDDFactory {
loadStatus: SegmentStatus,
overwriteTable: Boolean
): Boolean = {
+ val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val metadataDetails = if (status != null && status(0) != null) {
status(0)._2._1
} else {
@@ -758,6 +757,8 @@ object CarbonDataRDDFactory {
loadStatus,
carbonLoadModel.getFactTimeStamp,
true)
+ CarbonUtil
+ .addDataIndexSizeIntoMetaEntry(metadataDetails, carbonLoadModel.getSegmentId, carbonTable)
val done = CarbonLoaderUtil.recordLoadMetadata(metadataDetails, carbonLoadModel, false,
overwriteTable)
if (!done) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/589f126d/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
index b233c99..b61078b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonDescribeFormattedCommand.scala
@@ -30,6 +30,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.CarbonUtil
private[sql] case class CarbonDescribeFormattedCommand(
child: SparkPlan,
@@ -112,6 +113,15 @@ private[sql] case class CarbonDescribeFormattedCommand(
.getOrDefault(CarbonCommonConstants.TABLE_COMMENT, "")
results ++= Seq(("Comment: ", tableComment, ""))
results ++= Seq(("Table Block Size : ", carbonTable.getBlockSizeInMB + " MB", ""))
+ val dataIndexSize = CarbonUtil.calculateDataIndexSize(carbonTable)
+ if (!dataIndexSize.isEmpty) {
+ results ++= Seq((CarbonCommonConstants.TABLE_DATA_SIZE + ":",
+ dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE).toString, ""))
+ results ++= Seq((CarbonCommonConstants.TABLE_INDEX_SIZE + ":",
+ dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE).toString, ""))
+ results ++= Seq((CarbonCommonConstants.LAST_UPDATE_TIME + ":",
+ dataIndexSize.get(CarbonCommonConstants.LAST_UPDATE_TIME).toString, ""))
+ }
results ++= Seq(("SORT_SCOPE", carbonTable.getTableInfo.getFactTable
.getTableProperties.getOrDefault("sort_scope", CarbonCommonConstants
.LOAD_SORT_SCOPE_DEFAULT), CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/589f126d/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala b/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala
new file mode 100644
index 0000000..03ec3a1
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.scalatest.BeforeAndAfterAll
+
+class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll {
+ override def beforeAll(): Unit = {
+ sql("DROP TABLE IF EXISTS tableSize1")
+ sql("DROP TABLE IF EXISTS tableSize2")
+ sql("DROP TABLE IF EXISTS tableSize3")
+ sql("DROP TABLE IF EXISTS tableSize4")
+ sql("DROP TABLE IF EXISTS tableSize5")
+ sql("DROP TABLE IF EXISTS tableSize6")
+ sql("DROP TABLE IF EXISTS tableSize7")
+ sql("DROP TABLE IF EXISTS tableSize8")
+ sql("DROP TABLE IF EXISTS tableSize9")
+ sql("DROP TABLE IF EXISTS tableSize10")
+ sql("DROP TABLE IF EXISTS tableSize11")
+ }
+
+ override def afterAll(): Unit = {
+ sql("DROP TABLE IF EXISTS tableSize1")
+ sql("DROP TABLE IF EXISTS tableSize2")
+ sql("DROP TABLE IF EXISTS tableSize3")
+ sql("DROP TABLE IF EXISTS tableSize4")
+ sql("DROP TABLE IF EXISTS tableSize5")
+ sql("DROP TABLE IF EXISTS tableSize6")
+ sql("DROP TABLE IF EXISTS tableSize7")
+ sql("DROP TABLE IF EXISTS tableSize8")
+ sql("DROP TABLE IF EXISTS tableSize9")
+ sql("DROP TABLE IF EXISTS tableSize10")
+ sql("DROP TABLE IF EXISTS tableSize11")
+ }
+
+ test("get data size and index size after load data") {
+ sql("CREATE TABLE tableSize1 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize1 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ checkExistence(sql("DESCRIBE FORMATTED tableSize1"), true, CarbonCommonConstants.TABLE_DATA_SIZE)
+ checkExistence(sql("DESCRIBE FORMATTED tableSize1"), true, CarbonCommonConstants.TABLE_INDEX_SIZE)
+ val res1 = sql("DESCRIBE FORMATTED tableSize1").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
+ row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
+ assert(res1.length == 2)
+ res1.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ }
+
+ test("get data size and index size after major compaction") {
+ sql("CREATE TABLE tableSize2 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize2 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize2 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql("ALTER TABLE tableSize2 compact 'major'")
+ checkExistence(sql("DESCRIBE FORMATTED tableSize2"), true, CarbonCommonConstants.TABLE_DATA_SIZE)
+ checkExistence(sql("DESCRIBE FORMATTED tableSize2"), true, CarbonCommonConstants.TABLE_INDEX_SIZE)
+ val res2 = sql("DESCRIBE FORMATTED tableSize2").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
+ row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
+ assert(res2.length == 2)
+ res2.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ }
+
+ test("get data size and index size after minor compaction") {
+ sql("CREATE TABLE tableSize3 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql("ALTER TABLE tableSize3 compact 'minor'")
+ checkExistence(sql("DESCRIBE FORMATTED tableSize3"), true, CarbonCommonConstants.TABLE_DATA_SIZE)
+ checkExistence(sql("DESCRIBE FORMATTED tableSize3"), true, CarbonCommonConstants.TABLE_INDEX_SIZE)
+ val res3 = sql("DESCRIBE FORMATTED tableSize3").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
+ row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
+ assert(res3.length == 2)
+ res3.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ }
+
+ test("get data size and index size after insert into") {
+ sql("CREATE TABLE tableSize4 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize4 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql("CREATE TABLE tableSize5 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ sql("INSERT INTO TABLE tableSize5 SELECT * FROM tableSize4")
+ checkExistence(sql("DESCRIBE FORMATTED tableSize5"), true, CarbonCommonConstants.TABLE_DATA_SIZE)
+ checkExistence(sql("DESCRIBE FORMATTED tableSize5"), true, CarbonCommonConstants.TABLE_INDEX_SIZE)
+ val res4 = sql("DESCRIBE FORMATTED tableSize5").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
+ row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
+ assert(res4.length == 2)
+ res4.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ }
+
+ test("get data size and index size after insert overwrite") {
+ sql("CREATE TABLE tableSize6 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize6 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+ sql("CREATE TABLE tableSize7 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ sql("INSERT OVERWRITE TABLE tableSize7 SELECT * FROM tableSize6")
+ checkExistence(sql("DESCRIBE FORMATTED tableSize7"), true, CarbonCommonConstants.TABLE_DATA_SIZE)
+ checkExistence(sql("DESCRIBE FORMATTED tableSize7"), true, CarbonCommonConstants.TABLE_INDEX_SIZE)
+ val res5 = sql("DESCRIBE FORMATTED tableSize7").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
+ row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
+ assert(res5.length == 2)
+ res5.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ }
+
+ test("get data size and index size for empty table") {
+ sql("CREATE TABLE tableSize8 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ val res6 = sql("DESCRIBE FORMATTED tableSize8").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
+ row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
+ assert(res6.length == 2)
+ res6.foreach(row => assert(row.getString(1).trim.toLong == 0))
+ }
+
+ test("get last update time for empty table") {
+ sql("CREATE TABLE tableSize9 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ val res7 = sql("DESCRIBE FORMATTED tableSize9").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.LAST_UPDATE_TIME))
+ assert(res7.length == 1)
+ res7.foreach(row => assert(row.getString(1).trim.toLong == 0))
+ }
+
+ test("get last update time for unempty table") {
+ sql("CREATE TABLE tableSize10 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize10 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
+
+ val res8 = sql("DESCRIBE FORMATTED tableSize10").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.LAST_UPDATE_TIME))
+ assert(res8.length == 1)
+ res8.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ }
+
+ test("index and datasize for update scenario") {
+ sql(
+ "CREATE TABLE tableSize11 (empno int, workgroupcategory string, deptno int, projectcode " +
+ "int, attendance int) STORED BY 'org.apache.carbondata.format'")
+ sql(
+ s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize11 OPTIONS
+ |('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""".stripMargin)
+ val res9 = sql("DESCRIBE FORMATTED tableSize11").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
+ row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
+ assert(res9.length == 2)
+ res9.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ sql("update tableSize11 set (empno) = (234)").show()
+ val res10 = sql("DESCRIBE FORMATTED tableSize11").collect()
+ .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
+ row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
+ assert(res10.length == 2)
+ res10.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/589f126d/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 8f6d19c..15ee4fb 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -50,6 +50,7 @@ import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
@@ -281,8 +282,7 @@ public final class CarbonDataMergerUtil {
*/
public static boolean updateLoadMetadataWithMergeStatus(List<LoadMetadataDetails> loadsToMerge,
String metaDataFilepath, String mergedLoadNumber, CarbonLoadModel carbonLoadModel,
- long mergeLoadStartTime, CompactionType compactionType) {
-
+ long mergeLoadStartTime, CompactionType compactionType) throws IOException {
boolean tableStatusUpdationStatus = false;
AbsoluteTableIdentifier absoluteTableIdentifier =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
@@ -325,7 +325,10 @@ public final class CarbonDataMergerUtil {
loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS);
long loadEnddate = CarbonUpdateUtil.readCurrentTime();
loadMetadataDetails.setLoadEndTime(loadEnddate);
+ CarbonTable carbonTable = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable();
loadMetadataDetails.setLoadName(mergedLoadNumber);
+ CarbonUtil
+ .addDataIndexSizeIntoMetaEntry(loadMetadataDetails, mergedLoadNumber, carbonTable);
loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
loadMetadataDetails.setPartitionCount("0");
// if this is a major compaction then set the segment as major compaction.
[14/28] carbondata git commit: [CARBONDATA-1750] Fix NPE when
tablestatus file is empty
Posted by ja...@apache.org.
[CARBONDATA-1750] Fix NPE when tablestatus file is empty
Fix NPE when tablestatus file is empty
This closes #1517
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/52bf7c81
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/52bf7c81
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/52bf7c81
Branch: refs/heads/fgdatamap
Commit: 52bf7c81c8ecc632bdfaee6225a2e83ca697c475
Parents: c3e326e
Author: QiangCai <qi...@qq.com>
Authored: Fri Nov 17 10:45:13 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Nov 17 15:54:26 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/statusmanager/SegmentStatusManager.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/52bf7c81/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index 1944f96..2409219 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -205,11 +205,17 @@ public class SegmentStatusManager {
listOfLoadFolderDetailsArray =
gsonObjectToRead.fromJson(buffReader, LoadMetadataDetails[].class);
} catch (IOException e) {
+ LOG.error(e, "Failed to read metadata of load");
return new LoadMetadataDetails[0];
} finally {
closeStreams(buffReader, inStream, dataInputStream);
}
+ // if listOfLoadFolderDetailsArray is null, return empty array
+ if (null == listOfLoadFolderDetailsArray) {
+ return new LoadMetadataDetails[0];
+ }
+
return listOfLoadFolderDetailsArray;
}
[09/28] carbondata git commit: [CARBONDATA-1739] Clean up store path
interface
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index 153b169..07491d1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -65,7 +65,7 @@ object AlterTableUtil {
sys.error(s"Table $dbName.$tableName does not exist")
}
// acquire the lock first
- val table = relation.tableMeta.carbonTable
+ val table = relation.carbonTable
val acquiredLocks = ListBuffer[ICarbonLock]()
try {
locksToBeAcquired.foreach { lock =>
@@ -133,7 +133,7 @@ object AlterTableUtil {
thriftTable: TableInfo)(sparkSession: SparkSession,
sessionState: CarbonSessionState): Unit = {
val dbName = carbonTable.getDatabaseName
- val tableName = carbonTable.getFactTableName
+ val tableName = carbonTable.getTableName
CarbonEnv.getInstance(sparkSession).carbonMetastore
.updateTableSchemaForAlter(carbonTable.getCarbonTableIdentifier,
carbonTable.getCarbonTableIdentifier,
@@ -232,10 +232,7 @@ object AlterTableUtil {
def revertAddColumnChanges(dbName: String, tableName: String, timeStamp: Long)
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val carbonTable = metastore
- .lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation].tableMeta
- .carbonTable
-
+ val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath,
carbonTable.getCarbonTableIdentifier)
val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
@@ -262,9 +259,7 @@ object AlterTableUtil {
def revertDropColumnChanges(dbName: String, tableName: String, timeStamp: Long)
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val carbonTable = metastore
- .lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation].tableMeta
- .carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath,
carbonTable.getCarbonTableIdentifier)
val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
@@ -297,9 +292,7 @@ object AlterTableUtil {
def revertDataTypeChanges(dbName: String, tableName: String, timeStamp: Long)
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val carbonTable = metastore
- .lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation].tableMeta
- .carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath,
carbonTable.getCarbonTableIdentifier)
val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
@@ -343,30 +336,27 @@ object AlterTableUtil {
val locksToBeAcquired = List(LockUsage.METADATA_LOCK, LockUsage.COMPACTION_LOCK)
var locks = List.empty[ICarbonLock]
var timeStamp = 0L
- var newCols = Seq[org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema]()
var carbonTable: CarbonTable = null
try {
locks = AlterTableUtil
.validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- carbonTable = metastore
- .lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
- .tableMeta.carbonTable
+ carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
// get the latest carbon table
// read the latest schema file
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath,
carbonTable.getCarbonTableIdentifier)
val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
- val wrapperTableInfo = schemaConverter
- .fromExternalToWrapperTableInfo(thriftTableInfo,
- dbName,
- tableName,
- carbonTable.getTablePath)
+ val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
+ thriftTableInfo,
+ dbName,
+ tableName,
+ carbonTable.getTablePath)
val schemaEvolutionEntry = new org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry
schemaEvolutionEntry.setTimeStamp(timeStamp)
- val thriftTable = schemaConverter
- .fromWrapperToExternalTableInfo(wrapperTableInfo, dbName, tableName)
+ val thriftTable = schemaConverter.fromWrapperToExternalTableInfo(
+ wrapperTableInfo, dbName, tableName)
val tblPropertiesMap: mutable.Map[String, String] =
thriftTable.fact_table.getTableProperties.asScala
if (set) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
index dcfbaea..c05c0f1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
@@ -18,7 +18,6 @@
package org.apache.spark.util
import org.apache.spark.sql.{CarbonEnv, SparkSession}
-import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.api.CarbonStore
@@ -40,9 +39,7 @@ object CleanFiles {
def cleanFiles(spark: SparkSession, dbName: String, tableName: String,
storePath: String, forceTableClean: Boolean = false): Unit = {
TableAPIUtil.validateTableExists(spark, dbName, tableName)
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.
- lookupRelation(Some(dbName), tableName)(spark).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(spark)
CarbonStore.cleanFiles(dbName, tableName, storePath, carbonTable, forceTableClean)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
index 8375762..d682b21 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
@@ -17,7 +17,6 @@
package org.apache.spark.util
import org.apache.spark.sql.{CarbonEnv, SparkSession}
- import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.api.CarbonStore
@@ -30,9 +29,7 @@ object DeleteSegmentByDate {
def deleteSegmentByDate(spark: SparkSession, dbName: String, tableName: String,
dateValue: String): Unit = {
TableAPIUtil.validateTableExists(spark, dbName, tableName)
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.
- lookupRelation(Some(dbName), tableName)(spark).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(spark)
CarbonStore.deleteLoadByDate(dateValue, dbName, tableName, carbonTable)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
index 9b87504..5b58c8d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
@@ -34,9 +34,7 @@ object DeleteSegmentById {
def deleteSegmentById(spark: SparkSession, dbName: String, tableName: String,
segmentIds: Seq[String]): Unit = {
TableAPIUtil.validateTableExists(spark, dbName, tableName)
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.
- lookupRelation(Some(dbName), tableName)(spark).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(spark)
CarbonStore.deleteLoadById(segmentIds, dbName, tableName, carbonTable)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index 23cba20..287191c 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -245,7 +245,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
test("Alter table add partition: List Partition") {
sql("""ALTER TABLE list_table_area ADD PARTITION ('OutSpace', 'Hi')""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_list_table_area")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val partitionIds = partitionInfo.getPartitionIds
val list_info = partitionInfo.getListInfo
assert(partitionIds == List(0, 1, 2, 3, 4, 5).map(Integer.valueOf(_)).asJava)
@@ -286,7 +286,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("""ALTER TABLE list_table_area DROP PARTITION(2) WITH DATA""")
val carbonTable2 = CarbonMetadata.getInstance().getCarbonTable("default_list_table_area")
- val partitionInfo2 = carbonTable2.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo2 = carbonTable2.getPartitionInfo(carbonTable.getTableName)
val partitionIds2 = partitionInfo2.getPartitionIds
val list_info2 = partitionInfo2.getListInfo
assert(partitionIds2 == List(0, 1, 3, 4, 5).map(Integer.valueOf(_)).asJava)
@@ -304,7 +304,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
test("Alter table add partition: Range Partition") {
sql("""ALTER TABLE range_table_logdate ADD PARTITION ('2017/01/01', '2018/01/01')""")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_range_table_logdate")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val partitionIds = partitionInfo.getPartitionIds
val range_info = partitionInfo.getRangeInfo
assert(partitionIds == List(0, 1, 2, 3, 4, 5).map(Integer.valueOf(_)).asJava)
@@ -342,7 +342,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("""ALTER TABLE range_table_logdate DROP PARTITION(3) WITH DATA;""")
val carbonTable1 = CarbonMetadata.getInstance().getCarbonTable("default_range_table_logdate")
- val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable.getTableName)
val partitionIds1 = partitionInfo1.getPartitionIds
val range_info1 = partitionInfo1.getRangeInfo
assert(partitionIds1 == List(0, 1, 2, 4, 5).map(Integer.valueOf(_)).asJava)
@@ -373,7 +373,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
test("Alter table split partition: List Partition") {
sql("""ALTER TABLE list_table_country SPLIT PARTITION(4) INTO ('Canada', 'Russia', '(Good, NotGood)')""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_list_table_country")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val partitionIds = partitionInfo.getPartitionIds
val list_info = partitionInfo.getListInfo
assert(partitionIds == List(0, 1, 2, 3, 6, 7, 8, 5).map(Integer.valueOf(_)).asJava)
@@ -415,7 +415,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("""ALTER TABLE list_table_country DROP PARTITION(8)""")
val carbonTable1 = CarbonMetadata.getInstance().getCarbonTable("default_list_table_country")
- val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable.getTableName)
val partitionIds1 = partitionInfo1.getPartitionIds
val list_info1 = partitionInfo1.getListInfo
assert(partitionIds1 == List(0, 1, 2, 3, 6, 7, 5).map(Integer.valueOf(_)).asJava)
@@ -438,7 +438,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("""ALTER TABLE list_table_country ADD PARTITION ('(Part1, Part2, Part3, Part4)')""".stripMargin)
sql("""ALTER TABLE list_table_country SPLIT PARTITION(9) INTO ('Part4', 'Part2', '(Part1, Part3)')""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_list_table_country")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val partitionIds = partitionInfo.getPartitionIds
val list_info = partitionInfo.getListInfo
assert(partitionIds == List(0, 1, 2, 3, 6, 7, 5, 10, 11, 12).map(Integer.valueOf(_)).asJava)
@@ -485,7 +485,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("""ALTER TABLE list_table_area ADD PARTITION ('(One,Two, Three, Four)')""".stripMargin)
sql("""ALTER TABLE list_table_area SPLIT PARTITION(6) INTO ('One', '(Two, Three )', 'Four')""".stripMargin)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_list_table_area")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val partitionIds = partitionInfo.getPartitionIds
val list_info = partitionInfo.getListInfo
assert(partitionIds == List(0, 1, 3, 4, 5, 7, 8, 9).map(Integer.valueOf(_)).asJava)
@@ -528,7 +528,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
test("Alter table split partition: Range Partition") {
sql("""ALTER TABLE range_table_logdate_split SPLIT PARTITION(4) INTO ('2017/01/01', '2018/01/01')""")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_range_table_logdate_split")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val partitionIds = partitionInfo.getPartitionIds
val rangeInfo = partitionInfo.getRangeInfo
assert(partitionIds == List(0, 1, 2, 3, 5, 6).map(Integer.valueOf(_)).asJava)
@@ -566,7 +566,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("""ALTER TABLE range_table_logdate_split DROP PARTITION(6)""")
val carbonTable1 = CarbonMetadata.getInstance().getCarbonTable("default_range_table_logdate_split")
- val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable.getTableName)
val partitionIds1 = partitionInfo1.getPartitionIds
val rangeInfo1 = partitionInfo1.getRangeInfo
assert(partitionIds1 == List(0, 1, 2, 3, 5).map(Integer.valueOf(_)).asJava)
@@ -586,7 +586,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
test("Alter table split partition: Range Partition + Bucket") {
sql("""ALTER TABLE range_table_bucket SPLIT PARTITION(4) INTO ('2017/01/01', '2018/01/01')""")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_range_table_bucket")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val partitionIds = partitionInfo.getPartitionIds
val rangeInfo = partitionInfo.getRangeInfo
assert(partitionIds == List(0, 1, 2, 3, 5, 6).map(Integer.valueOf(_)).asJava)
@@ -624,7 +624,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("""ALTER TABLE range_table_bucket DROP PARTITION(6) WITH DATA""")
val carbonTable1 = CarbonMetadata.getInstance().getCarbonTable("default_range_table_bucket")
- val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable.getTableName)
val partitionIds1 = partitionInfo1.getPartitionIds
val rangeInfo1 = partitionInfo1.getRangeInfo
assert(partitionIds1 == List(0, 1, 2, 3, 5).map(Integer.valueOf(_)).asJava)
@@ -642,7 +642,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("""ALTER TABLE range_table_bucket DROP PARTITION(3)""")
val carbonTable2 = CarbonMetadata.getInstance().getCarbonTable("default_range_table_bucket")
- val partitionInfo2 = carbonTable2.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo2 = carbonTable2.getPartitionInfo(carbonTable.getTableName)
val partitionIds2 = partitionInfo2.getPartitionIds
val rangeInfo2 = partitionInfo2.getRangeInfo
assert(partitionIds2 == List(0, 1, 2, 5).map(Integer.valueOf(_)).asJava)
@@ -659,7 +659,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("""ALTER TABLE range_table_bucket DROP PARTITION(5)""")
val carbonTable3 = CarbonMetadata.getInstance().getCarbonTable("default_range_table_bucket")
- val partitionInfo3 = carbonTable3.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo3 = carbonTable3.getPartitionInfo(carbonTable.getTableName)
val partitionIds3 = partitionInfo3.getPartitionIds
val rangeInfo3 = partitionInfo3.getRangeInfo
assert(partitionIds3 == List(0, 1, 2).map(Integer.valueOf(_)).asJava)
@@ -789,7 +789,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("ALTER TABLE carbon_table_default_db ADD PARTITION ('2017')")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_carbon_table_default_db")
- val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ val partitionInfo = carbonTable.getPartitionInfo(carbonTable.getTableName)
val partitionIds = partitionInfo.getPartitionIds
val range_info = partitionInfo.getRangeInfo
assert(partitionIds == List(0, 1, 2, 3).map(Integer.valueOf(_)).asJava)
@@ -809,7 +809,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
sql("ALTER TABLE carbondb.carbontable ADD PARTITION ('2017')")
val carbonTable1 = CarbonMetadata.getInstance().getCarbonTable("carbondb_carbontable")
- val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable1.getFactTableName)
+ val partitionInfo1 = carbonTable1.getPartitionInfo(carbonTable1.getTableName)
val partitionIds1 = partitionInfo1.getPartitionIds
val range_info1 = partitionInfo1.getRangeInfo
assert(partitionIds1 == List(0, 1, 2, 3).map(Integer.valueOf(_)).asJava)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index a3024be..3f5d8c6 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -43,12 +43,12 @@ class AllDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll {
header: String,
allDictFilePath: String): CarbonLoadModel = {
val carbonLoadModel = new CarbonLoadModel
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
- val table = relation.tableMeta.carbonTable
+ carbonLoadModel.setTableName(relation.carbonTable.getDatabaseName)
+ carbonLoadModel.setDatabaseName(relation.carbonTable.getTableName)
+ val table = relation.carbonTable
val carbonSchema = new CarbonDataLoadSchema(table)
carbonLoadModel.setDatabaseName(table.getDatabaseName)
- carbonLoadModel.setTableName(table.getFactTableName)
+ carbonLoadModel.setTableName(table.getTableName)
carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
carbonLoadModel.setFactFilePath(filePath)
carbonLoadModel.setCsvHeader(header)
@@ -141,10 +141,8 @@ class AllDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll {
test("Support generate global dictionary from all dictionary files") {
val header = "id,name,city,age"
val carbonLoadModel = buildCarbonLoadModel(sampleRelation, null, header, sampleAllDictionaryFile)
- GlobalDictionaryUtil
- .generateGlobalDictionary(sqlContext,
- carbonLoadModel,
- sampleRelation.tableMeta.storePath)
+ GlobalDictionaryUtil.generateGlobalDictionary(
+ sqlContext, carbonLoadModel, sampleRelation.carbonTable.getTablePath)
DictionaryTestCaseUtil.
checkDictionary(sampleRelation, "city", "shenzhen")
@@ -156,7 +154,7 @@ class AllDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll {
GlobalDictionaryUtil
.generateGlobalDictionary(sqlContext,
carbonLoadModel,
- complexRelation.tableMeta.storePath)
+ complexRelation.carbonTable.getTablePath)
DictionaryTestCaseUtil.
checkDictionary(complexRelation, "channelsId", "1650")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
index 930de43..4551120 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
@@ -38,9 +38,9 @@ object DictionaryTestCaseUtil {
* @param value a value of column
*/
def checkDictionary(relation: CarbonRelation, columnName: String, value: String) {
- val table = relation.tableMeta.carbonTable
- val dimension = table.getDimensionByName(table.getFactTableName, columnName)
- val tableIdentifier = new CarbonTableIdentifier(table.getDatabaseName, table.getFactTableName, "uniqueid")
+ val table = relation.carbonTable
+ val dimension = table.getDimensionByName(table.getTableName, columnName)
+ val tableIdentifier = new CarbonTableIdentifier(table.getDatabaseName, table.getTableName, "uniqueid")
val absoluteTableIdentifier = new AbsoluteTableIdentifier(table.getTablePath, tableIdentifier)
val columnIdentifier = new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier,
dimension.getColumnIdentifier, dimension.getDataType,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index d37a68b..78ae384 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -151,12 +151,12 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
extColFilePath: String,
csvDelimiter: String = ","): CarbonLoadModel = {
val carbonLoadModel = new CarbonLoadModel
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
- val table = relation.tableMeta.carbonTable
+ carbonLoadModel.setTableName(relation.carbonTable.getDatabaseName)
+ carbonLoadModel.setDatabaseName(relation.carbonTable.getTableName)
+ val table = relation.carbonTable
val carbonSchema = new CarbonDataLoadSchema(table)
carbonLoadModel.setDatabaseName(table.getDatabaseName)
- carbonLoadModel.setTableName(table.getFactTableName)
+ carbonLoadModel.setTableName(table.getTableName)
carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
carbonLoadModel.setFactFilePath(filePath)
carbonLoadModel.setCsvHeader(header)
@@ -198,7 +198,7 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
var carbonLoadModel = buildCarbonLoadModel(extComplexRelation, complexFilePath1,
header, extColDictFilePath1)
GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
- extComplexRelation.tableMeta.storePath)
+ extComplexRelation.carbonTable.getTablePath)
// check whether the dictionary is generated
DictionaryTestCaseUtil.checkDictionary(
extComplexRelation, "deviceInformationId", "10086")
@@ -207,7 +207,7 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
carbonLoadModel = buildCarbonLoadModel(extComplexRelation, complexFilePath1,
header, extColDictFilePath2)
GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
- extComplexRelation.tableMeta.storePath)
+ extComplexRelation.carbonTable.getTablePath)
// check the old dictionary and whether the new distinct value is generated
DictionaryTestCaseUtil.checkDictionary(
extComplexRelation, "deviceInformationId", "10086")
@@ -220,7 +220,7 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
var carbonLoadModel = buildCarbonLoadModel(extComplexRelation, complexFilePath1,
header, extColDictFilePath3)
GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
- extComplexRelation.tableMeta.storePath)
+ extComplexRelation.carbonTable.getTablePath)
// check whether the dictionary is generated
DictionaryTestCaseUtil.checkDictionary(
extComplexRelation, "channelsId", "1421|")
@@ -229,7 +229,7 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
carbonLoadModel = buildCarbonLoadModel(verticalDelimiteRelation, complexFilePath2,
header2, extColDictFilePath3, "|")
GlobalDictionaryUtil.generateGlobalDictionary(sqlContext, carbonLoadModel,
- verticalDelimiteRelation.tableMeta.storePath)
+ verticalDelimiteRelation.carbonTable.getTablePath)
// check whether the dictionary is generated
DictionaryTestCaseUtil.checkDictionary(
verticalDelimiteRelation, "channelsId", "1431,")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index 442d93e..71c3dc2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -182,9 +182,9 @@ public final class DataLoadProcessBuilder {
loadModel.getBadRecordsLocation());
CarbonMetadata.getInstance().addCarbonTable(carbonTable);
List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
+ carbonTable.getDimensionByTableName(carbonTable.getTableName());
List<CarbonMeasure> measures =
- carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
+ carbonTable.getMeasureByTableName(carbonTable.getTableName());
Map<String, String> dateFormatMap =
CarbonDataProcessorUtil.getDateFormatMap(loadModel.getDateFormat());
List<DataField> dataFields = new ArrayList<>();
@@ -209,7 +209,7 @@ public final class DataLoadProcessBuilder {
}
}
configuration.setDataFields(dataFields.toArray(new DataField[dataFields.size()]));
- configuration.setBucketingInfo(carbonTable.getBucketingInfo(carbonTable.getFactTableName()));
+ configuration.setBucketingInfo(carbonTable.getBucketingInfo(carbonTable.getTableName()));
// configuration for one pass load: dictionary server info
configuration.setUseOnePass(loadModel.getUseOnePass());
configuration.setDictionaryServerHost(loadModel.getDictionaryServerHost());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
index be3572c..65f70a0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
@@ -209,7 +209,7 @@ public class CarbonCompactionExecutor {
List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
+ carbonTable.getDimensionByTableName(carbonTable.getTableName());
for (CarbonDimension dim : dimensions) {
// check if dimension is deleted
QueryDimension queryDimension = new QueryDimension(dim.getColName());
@@ -220,7 +220,7 @@ public class CarbonCompactionExecutor {
List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<CarbonMeasure> measures =
- carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
+ carbonTable.getMeasureByTableName(carbonTable.getTableName());
for (CarbonMeasure carbonMeasure : measures) {
// check if measure is deleted
QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index 08b8600..c60bb24 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -308,7 +308,7 @@ public class CarbonCompactionUtil {
public static int[] updateColumnSchemaAndGetCardinality(Map<String, Integer> columnCardinalityMap,
CarbonTable carbonTable, List<ColumnSchema> updatedColumnSchemaList) {
List<CarbonDimension> masterDimensions =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
+ carbonTable.getDimensionByTableName(carbonTable.getTableName());
List<Integer> updatedCardinalityList = new ArrayList<>(columnCardinalityMap.size());
for (CarbonDimension dimension : masterDimensions) {
Integer value = columnCardinalityMap.get(dimension.getColumnId());
@@ -321,7 +321,7 @@ public class CarbonCompactionUtil {
}
// add measures to the column schema list
List<CarbonMeasure> masterSchemaMeasures =
- carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
+ carbonTable.getMeasureByTableName(carbonTable.getTableName());
for (CarbonMeasure measure : masterSchemaMeasures) {
updatedColumnSchemaList.add(measure.getColumnSchema());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index c1df349..8f6d19c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -1264,7 +1264,7 @@ public final class CarbonDataMergerUtil {
lockStatus = carbonLock.lockWithRetries();
if (lockStatus) {
LOGGER.info(
- "Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName()
+ "Acquired lock for table" + table.getDatabaseName() + "." + table.getTableName()
+ " for table status updation");
LoadMetadataDetails[] listOfLoadFolderDetailsArray =
@@ -1284,18 +1284,18 @@ public final class CarbonDataMergerUtil {
}
} else {
LOGGER.error("Not able to acquire the lock for Table status updation for table " + table
- .getDatabaseName() + "." + table.getFactTableName());
+ .getDatabaseName() + "." + table.getTableName());
}
} finally {
if (lockStatus) {
if (carbonLock.unlock()) {
LOGGER.info(
"Table unlocked successfully after table status updation" + table.getDatabaseName()
- + "." + table.getFactTableName());
+ + "." + table.getTableName());
} else {
LOGGER.error(
"Unable to unlock Table lock for table" + table.getDatabaseName() + "." + table
- .getFactTableName() + " during table status updation");
+ .getTableName() + " during table status updation");
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/merger/TableMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/TableMeta.java b/processing/src/main/java/org/apache/carbondata/processing/merger/TableMeta.java
deleted file mode 100644
index 09dbfff..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/TableMeta.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.merger;
-
-import java.io.Serializable;
-
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-
-public class TableMeta implements Serializable {
-
- private static final long serialVersionUID = -1749874611119829431L;
-
- public CarbonTableIdentifier carbonTableIdentifier;
- public String storePath;
- public CarbonTable carbonTable;
- public String tablePath;
-
- public TableMeta(CarbonTableIdentifier carbonTableIdentifier, String storePath, String tablePath,
- CarbonTable carbonTable) {
- this.carbonTableIdentifier = carbonTableIdentifier;
- this.storePath = storePath;
- this.tablePath = tablePath;
- this.carbonTable = carbonTable;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
index aeddac6..36e022b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
@@ -78,7 +78,7 @@ public abstract class AbstractCarbonQueryExecutor {
List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
+ carbonTable.getDimensionByTableName(carbonTable.getTableName());
for (CarbonDimension dim : dimensions) {
// check if dimension is deleted
QueryDimension queryDimension = new QueryDimension(dim.getColName());
@@ -89,7 +89,7 @@ public abstract class AbstractCarbonQueryExecutor {
List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<CarbonMeasure> measures =
- carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
+ carbonTable.getMeasureByTableName(carbonTable.getTableName());
for (CarbonMeasure carbonMeasure : measures) {
// check if measure is deleted
QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
index 1db414f..48c5471 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
@@ -46,7 +46,7 @@ public class RowResultProcessor {
SegmentProperties segProp, String[] tempStoreLocation, Integer bucketId) {
CarbonDataProcessorUtil.createLocations(tempStoreLocation);
this.segmentProperties = segProp;
- String tableName = carbonTable.getFactTableName();
+ String tableName = carbonTable.getTableName();
CarbonFactDataHandlerModel carbonFactDataHandlerModel =
CarbonFactDataHandlerModel.getCarbonFactDataHandlerModel(loadModel, carbonTable,
segProp, tableName, tempStoreLocation);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 504e7ec..75fcea3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.processing.store;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -42,19 +41,15 @@ import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthEquiSplitGenerator;
import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.datatypes.GenericDataType;
import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
-import org.apache.carbondata.processing.store.file.FileManager;
-import org.apache.carbondata.processing.store.file.IFileManagerComposite;
import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
import org.apache.carbondata.processing.store.writer.CarbonFactDataWriter;
@@ -77,10 +72,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
private CarbonFactDataWriter dataWriter;
/**
- * File manager
- */
- private IFileManagerComposite fileManager;
- /**
* total number of entries in blocklet
*/
private int entryCount;
@@ -91,11 +82,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
*/
private int pageSize;
- // This variable is true if it is dictionary dimension and its cardinality is lower than
- // property of CarbonCommonConstants.HIGH_CARDINALITY_VALUE
- // It decides whether it will do RLE encoding on data page for this dimension
- private boolean[] rleEncodingForDictDimension;
- private boolean[] isNoDictionary;
private long processedDataCount;
private ExecutorService producerExecutorService;
private List<Future<Void>> producerExecutorServiceTaskList;
@@ -130,12 +116,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
private boolean processingComplete;
/**
- * boolean to check whether dimension
- * is of dictionary type or no dictionary type
- */
- private boolean[] isDictDimension;
-
- /**
* current data format version
*/
private ColumnarFormatVersion version;
@@ -146,47 +126,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
public CarbonFactDataHandlerColumnar(CarbonFactDataHandlerModel model) {
this.model = model;
initParameters(model);
-
- int numDimColumns = colGrpModel.getNoOfColumnStore() + model.getNoDictionaryCount()
- + getExpandedComplexColsCount();
- this.rleEncodingForDictDimension = new boolean[numDimColumns];
- this.isNoDictionary = new boolean[numDimColumns];
-
- int noDictStartIndex = this.colGrpModel.getNoOfColumnStore();
- // setting true value for dims of high card
- for (int i = 0; i < model.getNoDictionaryCount(); i++) {
- this.isNoDictionary[noDictStartIndex + i] = true;
- }
-
- boolean isAggKeyBlock = Boolean.parseBoolean(
- CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.AGGREAGATE_COLUMNAR_KEY_BLOCK,
- CarbonCommonConstants.AGGREAGATE_COLUMNAR_KEY_BLOCK_DEFAULTVALUE));
- if (isAggKeyBlock) {
- int[] dimLens = model.getSegmentProperties().getDimColumnsCardinality();
- for (int i = 0; i < model.getTableSpec().getNumSimpleDimensions(); i++) {
- if (model.getSegmentProperties().getDimensions().get(i).isGlobalDictionaryEncoding()) {
- this.rleEncodingForDictDimension[i] = true;
- }
- }
-
- if (model.getDimensionCount() < dimLens.length) {
- int allColsCount = getColsCount(model.getDimensionCount());
- List<Boolean> rleWithComplex = new ArrayList<Boolean>(allColsCount);
- for (int i = 0; i < model.getDimensionCount(); i++) {
- GenericDataType complexDataType = model.getComplexIndexMap().get(i);
- if (complexDataType != null) {
- complexDataType.fillAggKeyBlock(rleWithComplex, this.rleEncodingForDictDimension);
- } else {
- rleWithComplex.add(this.rleEncodingForDictDimension[i]);
- }
- }
- this.rleEncodingForDictDimension = new boolean[allColsCount];
- for (int i = 0; i < allColsCount; i++) {
- this.rleEncodingForDictDimension[i] = rleWithComplex.get(i);
- }
- }
- }
this.version = CarbonProperties.getInstance().getFormatVersion();
StringBuffer noInvertedIdxCol = new StringBuffer();
for (CarbonDimension cd : model.getSegmentProperties().getDimensions()) {
@@ -202,13 +141,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
SortScopeOptions.SortScope sortScope = model.getSortScope();
this.colGrpModel = model.getSegmentProperties().getColumnGroupModel();
- //TODO need to pass carbon table identifier to metadata
- CarbonTable carbonTable =
- CarbonMetadata.getInstance().getCarbonTable(
- model.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + model.getTableName());
- isDictDimension =
- CarbonUtil.identifyDimensionType(carbonTable.getDimensionByTableName(model.getTableName()));
-
// in compaction flow the measure with decimal type will come as spark decimal.
// need to convert it to byte array.
if (model.isCompactionFlow()) {
@@ -247,19 +179,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
consumerExecutorServiceTaskList.add(consumerExecutorService.submit(consumer));
}
- private boolean[] arrangeUniqueBlockType(boolean[] aggKeyBlock) {
- int counter = 0;
- boolean[] uniqueBlock = new boolean[aggKeyBlock.length];
- for (int i = 0; i < isDictDimension.length; i++) {
- if (isDictDimension[i]) {
- uniqueBlock[i] = aggKeyBlock[counter++];
- } else {
- uniqueBlock[i] = false;
- }
- }
- return uniqueBlock;
- }
-
private void setComplexMapSurrogateIndex(int dimensionCount) {
int surrIndex = 0;
for (int i = 0; i < dimensionCount; i++) {
@@ -283,9 +202,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
* @throws CarbonDataWriterException
*/
public void initialise() throws CarbonDataWriterException {
- fileManager = new FileManager();
- // todo: the fileManager seems to be useless, remove it later
- fileManager.setName(new File(model.getStoreLocation()[0]).getName());
setWritingConfiguration();
}
@@ -412,27 +328,11 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
}
- private int getColsCount(int columnSplit) {
- int count = 0;
- for (int i = 0; i < columnSplit; i++) {
- GenericDataType complexDataType = model.getComplexIndexMap().get(i);
- if (complexDataType != null) {
- count += complexDataType.getColsCount();
- } else count++;
- }
- return count;
- }
-
// return the number of complex column after complex columns are expanded
private int getExpandedComplexColsCount() {
return model.getExpandedComplexColsCount();
}
- // return the number of complex column
- private int getComplexColumnCount() {
- return model.getComplexIndexMap().size();
- }
-
/**
* below method will be used to close the handler
*/
@@ -519,7 +419,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
this.dataWriter = getFactDataWriter();
// initialize the channel;
this.dataWriter.initializeWriter();
- //initializeColGrpMinMax();
}
/**
@@ -571,14 +470,10 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
carbonDataWriterVo.setStoreLocation(model.getStoreLocation());
carbonDataWriterVo.setMeasureCount(model.getMeasureCount());
carbonDataWriterVo.setTableName(model.getTableName());
- carbonDataWriterVo.setFileManager(fileManager);
- carbonDataWriterVo.setRleEncodingForDictDim(rleEncodingForDictDimension);
- carbonDataWriterVo.setIsComplexType(isComplexTypes());
carbonDataWriterVo.setNoDictionaryCount(model.getNoDictionaryCount());
carbonDataWriterVo.setCarbonDataFileAttributes(model.getCarbonDataFileAttributes());
carbonDataWriterVo.setDatabaseName(model.getDatabaseName());
carbonDataWriterVo.setWrapperColumnSchemaList(model.getWrapperColumnSchema());
- carbonDataWriterVo.setIsDictionaryColumn(isDictDimension);
carbonDataWriterVo.setCarbonDataDirectoryPath(model.getCarbonDataDirectoryPath());
carbonDataWriterVo.setColCardinality(model.getColCardinality());
carbonDataWriterVo.setSegmentProperties(model.getSegmentProperties());
@@ -590,31 +485,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
return carbonDataWriterVo;
}
- private boolean[] isComplexTypes() {
- int noDictionaryCount = model.getNoDictionaryCount();
- int noOfColumn = colGrpModel.getNoOfColumnStore() + noDictionaryCount + getComplexColumnCount();
- int allColsCount = getColsCount(noOfColumn);
- boolean[] isComplexType = new boolean[allColsCount];
-
- List<Boolean> complexTypesList = new ArrayList<Boolean>(allColsCount);
- for (int i = 0; i < noOfColumn; i++) {
- GenericDataType complexDataType = model.getComplexIndexMap().get(i - noDictionaryCount);
- if (complexDataType != null) {
- int count = complexDataType.getColsCount();
- for (int j = 0; j < count; j++) {
- complexTypesList.add(true);
- }
- } else {
- complexTypesList.add(false);
- }
- }
- for (int i = 0; i < allColsCount; i++) {
- isComplexType[i] = complexTypesList.get(i);
- }
-
- return isComplexType;
- }
-
/**
* This method will reset the block processing count
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/store/file/FileData.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/file/FileData.java b/processing/src/main/java/org/apache/carbondata/processing/store/file/FileData.java
deleted file mode 100644
index ddd9bf2..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/store/file/FileData.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.store.file;
-
-
-public class FileData extends FileManager {
-
- /**
- * Store Path
- */
- private String storePath;
-
- /**
- * hierarchyValueWriter
- */
-
- public FileData(String fileName, String storePath) {
- this.fileName = fileName;
- this.storePath = storePath;
- }
-
- /**
- * @return Returns the carbonDataFileTempPath.
- */
- public String getFileName() {
- return fileName;
- }
-
- /**
- * @return Returns the storePath.
- */
- public String getStorePath() {
- return storePath;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/store/file/FileManager.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/file/FileManager.java b/processing/src/main/java/org/apache/carbondata/processing/store/file/FileManager.java
deleted file mode 100644
index cfa3a66..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/store/file/FileManager.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.store.file;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-
-public class FileManager implements IFileManagerComposite {
- /**
- * listOfFileData, composite parent which holds the different objects
- */
- protected List<IFileManagerComposite> listOfFileData =
- new ArrayList<IFileManagerComposite>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- protected String fileName;
-
- @Override public void add(IFileManagerComposite customData) {
- listOfFileData.add(customData);
- }
-
- @Override public void remove(IFileManagerComposite customData) {
- listOfFileData.remove(customData);
-
- }
-
- @Override public IFileManagerComposite get(int i) {
- return listOfFileData.get(i);
- }
-
- @Override public void setName(String name) {
- this.fileName = name;
- }
-
- /**
- * Return the size
- */
- public int size() {
- return listOfFileData.size();
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/store/file/IFileManagerComposite.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/file/IFileManagerComposite.java b/processing/src/main/java/org/apache/carbondata/processing/store/file/IFileManagerComposite.java
deleted file mode 100644
index 6691772..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/store/file/IFileManagerComposite.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.store.file;
-
-public interface IFileManagerComposite {
- /**
- * Add the data which can be either row Folder(Composite) or File
- *
- * @param customData
- */
- void add(IFileManagerComposite customData);
-
- /**
- * Remove the CustomData type object from the IFileManagerComposite object hierarchy.
- *
- * @param customData
- */
- void remove(IFileManagerComposite customData);
-
- /**
- * get the CustomData type object name
- *
- * @return CustomDataIntf type
- */
- IFileManagerComposite get(int i);
-
- /**
- * set the CustomData type object name
- *
- * @param name
- */
- void setName(String name);
-
- /**
- * Get the size
- *
- * @return
- */
- int size();
-
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 1b6ba72..855ec03 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -60,7 +60,6 @@ import org.apache.carbondata.format.BlockIndex;
import org.apache.carbondata.format.BlockletInfo3;
import org.apache.carbondata.format.IndexHeader;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
-import org.apache.carbondata.processing.store.file.FileData;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.io.IOUtils;
@@ -317,9 +316,6 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
.getCarbonDataFileName(fileCount, dataWriterVo.getCarbonDataFileAttributes().getTaskId(),
dataWriterVo.getBucketNumber(), dataWriterVo.getTaskExtension(),
"" + dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
- String actualFileNameVal = carbonDataFileName + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
- FileData fileData = new FileData(actualFileNameVal, chosenTempLocation);
- dataWriterVo.getFileManager().add(fileData);
this.carbonDataFileTempPath = chosenTempLocation + File.separator
+ carbonDataFileName + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
this.fileCount++;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java
index 26fff09..79cdd95 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java
@@ -22,7 +22,6 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
import org.apache.carbondata.processing.store.CarbonDataFileAttributes;
-import org.apache.carbondata.processing.store.file.IFileManagerComposite;
/**
* Value object for writing the data
@@ -35,12 +34,6 @@ public class CarbonDataWriterVo {
private String tableName;
- private IFileManagerComposite fileManager;
-
- private boolean[] rleEncodingForDictDim;
-
- private boolean[] isComplexType;
-
private int NoDictionaryCount;
private CarbonDataFileAttributes carbonDataFileAttributes;
@@ -49,8 +42,6 @@ public class CarbonDataWriterVo {
private List<ColumnSchema> wrapperColumnSchemaList;
- private boolean[] isDictionaryColumn;
-
private String carbonDataDirectoryPath;
private int[] colCardinality;
@@ -110,48 +101,6 @@ public class CarbonDataWriterVo {
}
/**
- * @return the fileManager
- */
- public IFileManagerComposite getFileManager() {
- return fileManager;
- }
-
- /**
- * @param fileManager the fileManager to set
- */
- public void setFileManager(IFileManagerComposite fileManager) {
- this.fileManager = fileManager;
- }
-
- /**
- * @return the rleEncodingForDictDim
- */
- public boolean[] getRleEncodingForDictDim() {
- return rleEncodingForDictDim;
- }
-
- /**
- * @param rleEncodingForDictDim the rleEncodingForDictDim to set
- */
- public void setRleEncodingForDictDim(boolean[] rleEncodingForDictDim) {
- this.rleEncodingForDictDim = rleEncodingForDictDim;
- }
-
- /**
- * @return the isComplexType
- */
- public boolean[] getIsComplexType() {
- return isComplexType;
- }
-
- /**
- * @param isComplexType the isComplexType to set
- */
- public void setIsComplexType(boolean[] isComplexType) {
- this.isComplexType = isComplexType;
- }
-
- /**
* @return the noDictionaryCount
*/
public int getNoDictionaryCount() {
@@ -208,20 +157,6 @@ public class CarbonDataWriterVo {
}
/**
- * @return the isDictionaryColumn
- */
- public boolean[] getIsDictionaryColumn() {
- return isDictionaryColumn;
- }
-
- /**
- * @param isDictionaryColumn the isDictionaryColumn to set
- */
- public void setIsDictionaryColumn(boolean[] isDictionaryColumn) {
- this.isDictionaryColumn = isDictionaryColumn;
- }
-
- /**
* @return the carbonDataDirectoryPath
*/
public String getCarbonDataDirectoryPath() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index ca40830..7218a12 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -383,7 +383,7 @@ public final class CarbonDataProcessorUtil {
*/
public static Set<String> getSchemaColumnNames(CarbonDataLoadSchema schema, String tableName) {
Set<String> columnNames = new HashSet<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- String factTableName = schema.getCarbonTable().getFactTableName();
+ String factTableName = schema.getCarbonTable().getTableName();
if (tableName.equals(factTableName)) {
List<CarbonDimension> dimensions =
schema.getCarbonTable().getDimensionByTableName(factTableName);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 29a979d..db3442e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -263,6 +263,11 @@ public final class CarbonLoaderUtil {
AbsoluteTableIdentifier absoluteTableIdentifier =
loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
+ String metadataPath = carbonTablePath.getMetadataDirectoryPath();
+ FileType fileType = FileFactory.getFileType(metadataPath);
+ if (!FileFactory.isFileExist(metadataPath, fileType)) {
+ FileFactory.mkdirs(metadataPath, fileType);
+ }
String tableStatusPath = carbonTablePath.getTableStatusFilePath();
SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
index 58cc019..e09e3db 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
@@ -297,9 +297,9 @@ public class StoreCreator {
String header = reader.readLine();
String[] split = header.split(",");
List<CarbonColumn> allCols = new ArrayList<CarbonColumn>();
- List<CarbonDimension> dims = table.getDimensionByTableName(table.getFactTableName());
+ List<CarbonDimension> dims = table.getDimensionByTableName(table.getTableName());
allCols.addAll(dims);
- List<CarbonMeasure> msrs = table.getMeasureByTableName(table.getFactTableName());
+ List<CarbonMeasure> msrs = table.getMeasureByTableName(table.getTableName());
allCols.addAll(msrs);
Set<String>[] set = new HashSet[dims.size()];
for (int i = 0; i < set.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
index 943858d..7682437 100644
--- a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
+++ b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
@@ -66,7 +66,7 @@ public class StreamSegment {
try {
if (carbonLock.lockWithRetries()) {
LOGGER.info(
- "Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName()
+ "Acquired lock for table" + table.getDatabaseName() + "." + table.getTableName()
+ " for stream table get or create segment");
LoadMetadataDetails[] details =
@@ -104,17 +104,17 @@ public class StreamSegment {
} else {
LOGGER.error(
"Not able to acquire the lock for stream table get or create segment for table " + table
- .getDatabaseName() + "." + table.getFactTableName());
+ .getDatabaseName() + "." + table.getTableName());
throw new IOException("Failed to get stream segment");
}
} finally {
if (carbonLock.unlock()) {
LOGGER.info("Table unlocked successfully after stream table get or create segment" + table
- .getDatabaseName() + "." + table.getFactTableName());
+ .getDatabaseName() + "." + table.getTableName());
} else {
LOGGER.error(
"Unable to unlock table lock for stream table" + table.getDatabaseName() + "." + table
- .getFactTableName() + " during stream table get or create segment");
+ .getTableName() + " during stream table get or create segment");
}
}
}
@@ -132,7 +132,7 @@ public class StreamSegment {
try {
if (carbonLock.lockWithRetries()) {
LOGGER.info(
- "Acquired lock for table" + table.getDatabaseName() + "." + table.getFactTableName()
+ "Acquired lock for table" + table.getDatabaseName() + "." + table.getTableName()
+ " for stream table finish segment");
LoadMetadataDetails[] details =
@@ -165,17 +165,17 @@ public class StreamSegment {
} else {
LOGGER.error(
"Not able to acquire the lock for stream table status updation for table " + table
- .getDatabaseName() + "." + table.getFactTableName());
+ .getDatabaseName() + "." + table.getTableName());
throw new IOException("Failed to get stream segment");
}
} finally {
if (carbonLock.unlock()) {
LOGGER.info(
"Table unlocked successfully after table status updation" + table.getDatabaseName()
- + "." + table.getFactTableName());
+ + "." + table.getTableName());
} else {
LOGGER.error("Unable to unlock Table lock for table" + table.getDatabaseName() + "." + table
- .getFactTableName() + " during table status updation");
+ .getTableName() + " during table status updation");
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index 31ed1f6..2c4d35f 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -147,7 +147,7 @@ object StreamSinkFactory {
val optionsFinal = DataLoadingUtil.getDataLoadingOptions(carbonProperty, parameters)
optionsFinal.put("sort_scope", "no_sort")
if (parameters.get("fileheader").isEmpty) {
- optionsFinal.put("fileheader", carbonTable.getCreateOrderColumn(carbonTable.getFactTableName)
+ optionsFinal.put("fileheader", carbonTable.getCreateOrderColumn(carbonTable.getTableName)
.asScala.map(_.getColName).mkString(","))
}
val carbonLoadModel = new CarbonLoadModel()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5fc7f06f/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonStreamingQueryListener.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonStreamingQueryListener.scala b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonStreamingQueryListener.scala
index 6ee3296..c2789f4 100644
--- a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonStreamingQueryListener.scala
+++ b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonStreamingQueryListener.scala
@@ -42,14 +42,14 @@ class CarbonStreamingQueryListener(spark: SparkSession) extends StreamingQueryLi
LockUsage.STREAMING_LOCK)
if (lock.lockWithRetries()) {
LOGGER.info("Acquired the lock for stream table: " + carbonTable.getDatabaseName + "." +
- carbonTable.getFactTableName)
+ carbonTable.getTableName)
cache.put(event.id, lock)
} else {
LOGGER.error("Not able to acquire the lock for stream table:" +
- carbonTable.getDatabaseName + "." + carbonTable.getFactTableName)
+ carbonTable.getDatabaseName + "." + carbonTable.getTableName)
throw new InterruptedException(
"Not able to acquire the lock for stream table: " + carbonTable.getDatabaseName + "." +
- carbonTable.getFactTableName)
+ carbonTable.getTableName)
}
}
}
[16/28] carbondata git commit: [CARBONDATA-1745] Use default
metastore path from Hive
Posted by ja...@apache.org.
[CARBONDATA-1745] Use default metastore path from Hive
This closes #1513
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dfc7442a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dfc7442a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dfc7442a
Branch: refs/heads/fgdatamap
Commit: dfc7442a483d839282edb6e1305c191aa60da65a
Parents: 75ec79e
Author: Jacky Li <ja...@qq.com>
Authored: Fri Nov 17 15:01:43 2017 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Fri Nov 17 16:25:59 2017 +0800
----------------------------------------------------------------------
.../core/metadata/AbsoluteTableIdentifier.java | 1 +
.../carbondata/examples/AlterTableExample.scala | 3 +-
.../examples/CarbonDataFrameExample.scala | 1 -
.../examples/CarbonPartitionExample.scala | 1 -
.../carbondata/examples/ExampleUtils.scala | 1 -
.../org/apache/spark/sql/CarbonSession.scala | 38 ++++++++++----------
.../spark/sql/hive/cli/CarbonSQLCLIDriver.scala | 3 +-
7 files changed, 23 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfc7442a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
index d5434d8..603a1c1 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
@@ -62,6 +62,7 @@ public class AbsoluteTableIdentifier implements Serializable {
CarbonTableIdentifier identifier = new CarbonTableIdentifier(dbName, tableName, "");
return new AbsoluteTableIdentifier(tablePath, identifier);
}
+
public String getTablePath() {
return tablePath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfc7442a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
index dd2a28a..472dc44 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
@@ -37,7 +37,6 @@ object AlterTableExample {
val storeLocation = s"$rootPath/examples/spark2/target/store"
val warehouse = s"$rootPath/examples/spark2/target/warehouse"
- val metastoredb = s"$rootPath/examples/spark2/target/metastore_db"
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
@@ -49,7 +48,7 @@ object AlterTableExample {
.master("local")
.appName("AlterTableExample")
.config("spark.sql.warehouse.dir", warehouse)
- .getOrCreateCarbonSession(storeLocation, metastoredb)
+ .getOrCreateCarbonSession(storeLocation)
spark.sparkContext.setLogLevel("WARN")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfc7442a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
index ac198d8..2450b49 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
@@ -31,7 +31,6 @@ object CarbonDataFrameExample {
+ "../../../..").getCanonicalPath
val storeLocation = s"$rootPath/examples/spark2/target/store"
val warehouse = s"$rootPath/examples/spark2/target/warehouse"
- val metastoredb = s"$rootPath/examples/spark2/target"
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfc7442a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
index d8aca6b..6837c56 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala
@@ -34,7 +34,6 @@ object CarbonPartitionExample {
+ "../../../..").getCanonicalPath
val storeLocation = s"$rootPath/examples/spark2/target/store"
val warehouse = s"$rootPath/examples/spark2/target/warehouse"
- val metastoredb = s"$rootPath/examples/spark2/target"
val testData = s"$rootPath/integration/spark-common-test/src/test/resources/partition_data.csv"
CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfc7442a/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala
index a48ed6a..7975859 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala
@@ -37,7 +37,6 @@ object ExampleUtils {
+ "../../../..").getCanonicalPath
val storeLocation = s"$rootPath/examples/spark2/target/store"
val warehouse = s"$rootPath/examples/spark2/target/warehouse"
- val metastoredb = s"$rootPath/examples/spark2/target"
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfc7442a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 88cbc36..21840e4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -66,7 +66,7 @@ class CarbonSession(@transient val sc: SparkContext,
}
if (existingSharedState.isEmpty) {
- CarbonSession.initListeners
+ CarbonSession.initListeners()
}
}
@@ -76,9 +76,7 @@ object CarbonSession {
implicit class CarbonBuilder(builder: Builder) {
def getOrCreateCarbonSession(): SparkSession = {
- getOrCreateCarbonSession(
- null,
- new File(CarbonCommonConstants.METASTORE_LOCATION_DEFAULT_VAL).getCanonicalPath)
+ getOrCreateCarbonSession(null, null)
}
def getOrCreateCarbonSession(storePath: String): SparkSession = {
@@ -94,17 +92,20 @@ object CarbonSession {
getValue("options", builder).asInstanceOf[scala.collection.mutable.HashMap[String, String]]
val userSuppliedContext: Option[SparkContext] =
getValue("userSuppliedContext", builder).asInstanceOf[Option[SparkContext]]
- val hadoopConf = new Configuration()
- val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
- if (configFile != null) {
- hadoopConf.addResource(configFile)
- }
- if (options.get(CarbonCommonConstants.HIVE_CONNECTION_URL).isEmpty &&
- hadoopConf.get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
- val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
- val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
- options ++= Map[String, String]((CarbonCommonConstants.HIVE_CONNECTION_URL,
- s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"))
+
+ if (metaStorePath != null) {
+ val hadoopConf = new Configuration()
+ val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
+ if (configFile != null) {
+ hadoopConf.addResource(configFile)
+ }
+ if (options.get(CarbonCommonConstants.HIVE_CONNECTION_URL).isEmpty &&
+ hadoopConf.get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
+ val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
+ val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
+ options ++= Map[String, String]((CarbonCommonConstants.HIVE_CONNECTION_URL,
+ s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"))
+ }
}
// Get the session from current thread's active session.
@@ -157,15 +158,16 @@ object CarbonSession {
}
sc
}
+
+ session = new CarbonSession(sparkContext)
val carbonProperties = CarbonProperties.getInstance()
if (storePath != null) {
carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION, storePath)
// In case if it is in carbon.properties for backward compatible
} else if (carbonProperties.getProperty(CarbonCommonConstants.STORE_LOCATION) == null) {
carbonProperties.addProperty(CarbonCommonConstants.STORE_LOCATION,
- sparkContext.conf.get("spark.sql.warehouse.dir"))
+ session.sessionState.conf.warehousePath)
}
- session = new CarbonSession(sparkContext)
options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
SparkSession.setDefaultSession(session)
CommonUtil.cleanInProgressSegments(
@@ -182,7 +184,7 @@ object CarbonSession {
session.streams.addListener(new CarbonStreamingQueryListener(session))
}
- return session
+ session
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dfc7442a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
index 22c7218..733744f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
@@ -39,7 +39,6 @@ object CarbonSQLCLIDriver {
def init() {
if (hiveContext == null) {
- val sparkConf = new SparkConf(loadDefaults = true)
import org.apache.spark.sql.CarbonSession._
@@ -50,7 +49,7 @@ object CarbonSQLCLIDriver {
.master(System.getProperty("spark.master"))
.appName("CarbonSQLCLIDriver")
.config("spark.sql.warehouse.dir", warehouse)
- .getOrCreateCarbonSession(storePath, storePath)
+ .getOrCreateCarbonSession(storePath)
hiveContext = carbon.sqlContext
hiveContext.conf.getAllConfs.toSeq.sorted.foreach { case (k, v) =>
[25/28] carbondata git commit: [CARBONDATA-1762] Remove existing
column level dateformat and support dateformat,
timestampformat in the load option
Posted by ja...@apache.org.
[CARBONDATA-1762] Remove existing column level dateformat and support dateformat, timestampformat in the load option
(1) Remove column level dateformat option
(2) Support dateformat and timestampformat in load options(table level)
This closes #1524
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/85dc4fff
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/85dc4fff
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/85dc4fff
Branch: refs/heads/fgdatamap
Commit: 85dc4fff0ecca160654085379310a1c3096731f7
Parents: 2a12938
Author: akashrn5 <ak...@gmail.com>
Authored: Fri Nov 17 16:55:33 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Nov 18 16:34:43 2017 +0800
----------------------------------------------------------------------
.../constants/CarbonLoadOptionConstants.java | 10 +++++-
.../carbondata/core/util/DataTypeUtil.java | 2 +-
.../TestLoadDataWithDiffTimestampFormat.scala | 31 ++++++++--------
.../carbondata/spark/load/ValidateUtil.scala | 38 +++++++++-----------
.../carbondata/spark/util/DataLoadingUtil.scala | 13 ++++++-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 24 +++++++------
.../processing/loading/DataField.java | 10 ++++++
.../loading/DataLoadProcessBuilder.java | 10 +++---
.../DirectDictionaryFieldConverterImpl.java | 10 +++++-
.../impl/NonDictionaryFieldConverterImpl.java | 8 ++++-
.../loading/model/CarbonLoadModel.java | 13 +++++++
.../util/CarbonDataProcessorUtil.java | 19 ----------
13 files changed, 114 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index ac278d9..e78d125 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -46,12 +46,20 @@ public final class CarbonLoadOptionConstants {
public static final String CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT = "false";
/**
- * option to specify the load option
+ * option to specify the dateFormat in load option for all date columns in table
*/
@CarbonProperty
public static final String CARBON_OPTIONS_DATEFORMAT =
"carbon.options.dateformat";
public static final String CARBON_OPTIONS_DATEFORMAT_DEFAULT = "";
+
+ /**
+ * option to specify the timestampFormat in load option for all timestamp columns in table
+ */
+ @CarbonProperty
+ public static final String CARBON_OPTIONS_TIMESTAMPFORMAT =
+ "carbon.options.timestampformat";
+ public static final String CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT = "";
/**
* option to specify the sort_scope
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 0961a63..3a25988 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -319,7 +319,7 @@ public final class DataTypeUtil {
Date dateToStr = null;
DateFormat dateFormatter = null;
try {
- if (null != dateFormat) {
+ if (null != dateFormat && !dateFormat.trim().isEmpty()) {
dateFormatter = new SimpleDateFormat(dateFormat);
} else {
dateFormatter = timeStampformatter.get();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
index 71d6466..906f05a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
@@ -17,10 +17,12 @@
package org.apache.carbondata.spark.testsuite.dataload
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
+import java.text.SimpleDateFormat
import org.apache.spark.sql.Row
import org.scalatest.BeforeAndAfterAll
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
@@ -31,26 +33,25 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
sql("DROP TABLE IF EXISTS t3")
sql("""
CREATE TABLE IF NOT EXISTS t3
- (ID Int, date Timestamp, starttime Timestamp, country String,
+ (ID Int, date date, starttime Timestamp, country String,
name String, phonetype String, serialname String, salary Int)
STORED BY 'carbondata'
""")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
}
test("test load data with different timestamp format") {
sql(s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
- OPTIONS('dateformat' = 'starttime:yyyy-MM-dd HH:mm:ss')
+ OPTIONS('dateformat' = 'yyyy/MM/dd','timestampformat'='yyyy-MM-dd HH:mm:ss')
""")
sql(s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData2.csv' into table t3
- OPTIONS('dateformat' = ' date : yyyy-MM-dd , StartTime : yyyy/MM/dd HH:mm:ss')
+ OPTIONS('dateformat' = 'yyyy-MM-dd','timestampformat'='yyyy/MM/dd HH:mm:ss')
""")
+ val sdf = new SimpleDateFormat("yyyy-MM-dd")
checkAnswer(
sql("SELECT date FROM t3 WHERE ID = 1"),
- Seq(Row(Timestamp.valueOf("2015-07-23 00:00:00.0")))
+ Seq(Row(new Date(sdf.parse("2015-07-23").getTime)))
)
checkAnswer(
sql("SELECT starttime FROM t3 WHERE ID = 1"),
@@ -58,7 +59,7 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
)
checkAnswer(
sql("SELECT date FROM t3 WHERE ID = 18"),
- Seq(Row(Timestamp.valueOf("2015-07-25 00:00:00.0")))
+ Seq(Row(new Date(sdf.parse("2015-07-25").getTime)))
)
checkAnswer(
sql("SELECT starttime FROM t3 WHERE ID = 18"),
@@ -75,19 +76,19 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Option DateFormat is not provided for Column date.")
+ assertResult(ex.getMessage)("Error: Wrong option: date is provided for option DateFormat")
case _: Throwable=> assert(false)
}
try {
sql(s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
- OPTIONS('dateformat' = 'fasfdas:yyyy/MM/dd')
+ OPTIONS('timestampformat' = 'timestamp')
""")
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Wrong Column Name fasfdas is provided in Option DateFormat.")
+ assertResult(ex.getMessage)("Error: Wrong option: timestamp is provided for option TimestampFormat")
case _: Throwable => assert(false)
}
@@ -99,7 +100,7 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Option DateFormat is not provided for Column date.")
+ assertResult(ex.getMessage)("Error: Wrong option: date: is provided for option DateFormat")
case _: Throwable => assert(false)
}
@@ -111,19 +112,19 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Option DateFormat is not provided for Column date .")
+ assertResult(ex.getMessage)("Error: Wrong option: date is provided for option DateFormat")
case _: Throwable => assert(false)
}
try {
sql(s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
- OPTIONS('dateformat' = ':yyyy/MM/dd ')
+ OPTIONS('dateformat' = 'fasfdas:yyyy/MM/dd')
""")
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Wrong Column Name is provided in Option DateFormat.")
+ assertResult(ex.getMessage)("Error: Wrong option: fasfdas:yyyy/MM/dd is provided for option DateFormat")
case _: Throwable => assert(false)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
index 51e0cc4..dfda92c 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
@@ -17,35 +17,31 @@
package org.apache.carbondata.spark.load
-import scala.collection.JavaConverters._
+import java.text.SimpleDateFormat
-import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.loading.sort.SortScopeOptions
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
object ValidateUtil {
- def validateDateFormat(dateFormat: String, table: CarbonTable, tableName: String): Unit = {
- val dimensions = table.getDimensionByTableName(tableName).asScala
+
+ /**
+ * validates both timestamp and date for illegal values
+ *
+ * @param dateTimeLoadFormat
+ * @param dateTimeLoadOption
+ */
+ def validateDateTimeFormat(dateTimeLoadFormat: String, dateTimeLoadOption: String): Unit = {
// allowing empty value to be configured for dateformat option.
- if (dateFormat != null && dateFormat.trim != "") {
- val dateFormats: Array[String] = dateFormat.split(CarbonCommonConstants.COMMA)
- for (singleDateFormat <- dateFormats) {
- val dateFormatSplits: Array[String] = singleDateFormat.split(":", 2)
- val columnName = dateFormatSplits(0).trim.toLowerCase
- if (!dimensions.exists(_.getColName.equals(columnName))) {
- throw new MalformedCarbonCommandException("Error: Wrong Column Name " +
- dateFormatSplits(0) +
- " is provided in Option DateFormat.")
- }
- if (dateFormatSplits.length < 2 || dateFormatSplits(1).trim.isEmpty) {
- throw new MalformedCarbonCommandException("Error: Option DateFormat is not provided " +
- "for " + "Column " + dateFormatSplits(0) +
- ".")
- }
- }
+ if (dateTimeLoadFormat != null && dateTimeLoadFormat.trim != "") {
+ try {
+ new SimpleDateFormat(dateTimeLoadFormat)
+ } catch {
+ case _: IllegalArgumentException =>
+ throw new MalformedCarbonCommandException(s"Error: Wrong option: $dateTimeLoadFormat is" +
+ s" provided for option $dateTimeLoadOption")
}
+ }
}
def validateSortScope(carbonTable: CarbonTable, sortScope: String): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 84ad85e..271962b 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -104,6 +104,14 @@ object DataLoadingUtil {
CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)))
optionsFinal.put(
+ "timestampformat",
+ options.getOrElse(
+ "timestampformat",
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT)))
+
+ optionsFinal.put(
"global_sort_partitions",
options.getOrElse(
"global_sort_partitions",
@@ -193,13 +201,15 @@ object DataLoadingUtil {
val bad_records_action = optionsFinal("bad_records_action")
val bad_record_path = optionsFinal("bad_record_path")
val global_sort_partitions = optionsFinal("global_sort_partitions")
+ val timestampformat = optionsFinal("timestampformat")
val dateFormat = optionsFinal("dateformat")
val delimeter = optionsFinal("delimiter")
val complex_delimeter_level1 = optionsFinal("complex_delimiter_level_1")
val complex_delimeter_level2 = optionsFinal("complex_delimiter_level_2")
val all_dictionary_path = optionsFinal("all_dictionary_path")
val column_dict = optionsFinal("columndict")
- ValidateUtil.validateDateFormat(dateFormat, table, table.getTableName)
+ ValidateUtil.validateDateTimeFormat(timestampformat, "TimestampFormat")
+ ValidateUtil.validateDateTimeFormat(dateFormat, "DateFormat")
ValidateUtil.validateSortScope(table, sort_scope)
if (bad_records_logger_enable.toBoolean ||
@@ -242,6 +252,7 @@ object DataLoadingUtil {
}
}
+ carbonLoadModel.setTimestampformat(timestampformat)
carbonLoadModel.setDateFormat(dateFormat)
carbonLoadModel.setDefaultTimestampFormat(carbonProperty.getProperty(
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index b4cde9b..b7f5def 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -854,7 +854,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
"SERIALIZATION_NULL_FORMAT", "BAD_RECORDS_LOGGER_ENABLE", "BAD_RECORDS_ACTION",
"ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "DATEFORMAT", "BAD_RECORD_PATH",
"BATCH_SORT_SIZE_INMB", "GLOBAL_SORT_PARTITIONS", "SINGLE_PASS",
- "IS_EMPTY_DATA_BAD_RECORD", "HEADER"
+ "IS_EMPTY_DATA_BAD_RECORD", "HEADER", "TIMESTAMPFORMAT"
)
var isSupported = true
val invalidOptions = StringBuilder.newBuilder
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 7dad243..38bb936 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -800,18 +800,20 @@ object CarbonDataRDDFactory {
throw new DataLoadingException("Partition column not found.")
}
- val dateFormatMap = CarbonDataProcessorUtil.getDateFormatMap(carbonLoadModel.getDateFormat)
- val specificFormat = Option(dateFormatMap.get(partitionColumn.toLowerCase))
- val timeStampFormat = if (specificFormat.isDefined) {
- new SimpleDateFormat(specificFormat.get)
- } else {
- val timestampFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
- .CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- new SimpleDateFormat(timestampFormatString)
- }
+ val specificTimestampFormat = carbonLoadModel.getTimestampformat
+ val specificDateFormat = carbonLoadModel.getDateFormat
+ val timeStampFormat =
+ if (specificTimestampFormat != null && !specificTimestampFormat.trim.isEmpty) {
+ new SimpleDateFormat(specificTimestampFormat)
+ } else {
+ val timestampFormatString = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ new SimpleDateFormat(timestampFormatString)
+ }
- val dateFormat = if (specificFormat.isDefined) {
- new SimpleDateFormat(specificFormat.get)
+ val dateFormat = if (specificDateFormat != null && !specificDateFormat.trim.isEmpty) {
+ new SimpleDateFormat(specificDateFormat)
} else {
val dateFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
index 5c81bb9..fb78deb 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
@@ -35,6 +35,8 @@ public class DataField implements Serializable {
private String dateFormat;
+ private String timestampFormat;
+
public boolean hasDictionaryEncoding() {
return column.hasEncoding(Encoding.DICTIONARY);
}
@@ -50,4 +52,12 @@ public class DataField implements Serializable {
public void setDateFormat(String dateFormat) {
this.dateFormat = dateFormat;
}
+
+ public String getTimestampFormat() {
+ return timestampFormat;
+ }
+
+ public void setTimestampFormat(String timestampFormat) {
+ this.timestampFormat = timestampFormat;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index 71c3dc2..8118e2f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -20,7 +20,6 @@ package org.apache.carbondata.processing.loading;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -28,6 +27,7 @@ import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -185,8 +185,6 @@ public final class DataLoadProcessBuilder {
carbonTable.getDimensionByTableName(carbonTable.getTableName());
List<CarbonMeasure> measures =
carbonTable.getMeasureByTableName(carbonTable.getTableName());
- Map<String, String> dateFormatMap =
- CarbonDataProcessorUtil.getDateFormatMap(loadModel.getDateFormat());
List<DataField> dataFields = new ArrayList<>();
List<DataField> complexDataFields = new ArrayList<>();
@@ -194,7 +192,11 @@ public final class DataLoadProcessBuilder {
// And then add complex data types and measures.
for (CarbonColumn column : dimensions) {
DataField dataField = new DataField(column);
- dataField.setDateFormat(dateFormatMap.get(column.getColName()));
+ if (column.getDataType() == DataTypes.DATE) {
+ dataField.setDateFormat(loadModel.getDateFormat());
+ } else if (column.getDataType() == DataTypes.TIMESTAMP) {
+ dataField.setTimestampFormat(loadModel.getTimestampformat());
+ }
if (column.isComplex()) {
complexDataFields.add(dataField);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
index 24c2f00..7dcef81 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.converter.BadRecordLogHolder;
@@ -42,11 +43,18 @@ public class DirectDictionaryFieldConverterImpl extends AbstractDictionaryFieldC
boolean isEmptyBadRecord) {
this.nullFormat = nullFormat;
this.column = dataField.getColumn();
- if (dataField.getDateFormat() != null && !dataField.getDateFormat().isEmpty()) {
+ if (dataField.getColumn().getDataType() == DataTypes.DATE && dataField.getDateFormat() != null
+ && !dataField.getDateFormat().isEmpty()) {
this.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dataField.getColumn().getDataType(),
dataField.getDateFormat());
+ } else if (dataField.getColumn().getDataType() == DataTypes.TIMESTAMP
+ && dataField.getTimestampFormat() != null && !dataField.getTimestampFormat().isEmpty()) {
+ this.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(dataField.getColumn().getDataType(),
+ dataField.getTimestampFormat());
+
} else {
this.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dataField.getColumn().getDataType());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
index eac9d69..e606cdb 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -60,10 +60,16 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
} else if (dimensionValue == null || dimensionValue.equals(nullformat)) {
updateWithNullValue(row);
} else {
+ String dateFormat = null;
+ if (dataType == DataTypes.DATE) {
+ dateFormat = dataField.getDateFormat();
+ } else if (dataType == DataTypes.TIMESTAMP) {
+ dateFormat = dataField.getTimestampFormat();
+ }
try {
row.update(DataTypeUtil
.getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType,
- dataField.getDateFormat()), index);
+ dateFormat), index);
} catch (Throwable ex) {
if (dimensionValue.length() > 0 || (dimensionValue.length() == 0 && isEmptyBadRecord)) {
String message = logHolder.getColumnMessageMap().get(column.getColName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index 8c3fe56..326ee62 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -93,6 +93,8 @@ public class CarbonLoadModel implements Serializable {
*/
private String commentChar;
+ private String timestampformat;
+
private String dateFormat;
private String defaultTimestampFormat;
@@ -351,6 +353,7 @@ public class CarbonLoadModel implements Serializable {
copy.escapeChar = escapeChar;
copy.quoteChar = quoteChar;
copy.commentChar = commentChar;
+ copy.timestampformat = timestampformat;
copy.dateFormat = dateFormat;
copy.defaultTimestampFormat = defaultTimestampFormat;
copy.maxColumns = maxColumns;
@@ -399,6 +402,7 @@ public class CarbonLoadModel implements Serializable {
copy.escapeChar = escapeChar;
copy.quoteChar = quoteChar;
copy.commentChar = commentChar;
+ copy.timestampformat = timestampformat;
copy.dateFormat = dateFormat;
copy.defaultTimestampFormat = defaultTimestampFormat;
copy.maxColumns = maxColumns;
@@ -449,6 +453,7 @@ public class CarbonLoadModel implements Serializable {
copyObj.escapeChar = escapeChar;
copyObj.quoteChar = quoteChar;
copyObj.commentChar = commentChar;
+ copyObj.timestampformat = timestampformat;
copyObj.dateFormat = dateFormat;
copyObj.defaultTimestampFormat = defaultTimestampFormat;
copyObj.maxColumns = maxColumns;
@@ -763,4 +768,12 @@ public class CarbonLoadModel implements Serializable {
public void setBadRecordsLocation(String badRecordsLocation) {
this.badRecordsLocation = badRecordsLocation;
}
+
+ public String getTimestampformat() {
+ return timestampformat;
+ }
+
+ public void setTimestampformat(String timestampformat) {
+ this.timestampformat = timestampformat;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 7218a12..b64362e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -21,7 +21,6 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -431,24 +430,6 @@ public final class CarbonDataProcessorUtil {
}
/**
- * Creates map for columns which dateformats mentioned while loading the data.
- * @param dataFormatString
- * @return
- */
- public static Map<String, String> getDateFormatMap(String dataFormatString) {
- Map<String, String> dateformatsHashMap = new HashMap<>();
- if (dataFormatString != null && !dataFormatString.isEmpty()) {
- String[] dateformats = dataFormatString.split(CarbonCommonConstants.COMMA);
- for (String dateFormat : dateformats) {
- String[] dateFormatSplits = dateFormat.split(":", 2);
- dateformatsHashMap
- .put(dateFormatSplits[0].toLowerCase().trim(), dateFormatSplits[1].trim());
- }
- }
- return dateformatsHashMap;
- }
-
- /**
* This method will get the store location for the given path, segment id and partition id
*
* @return data directory path
[02/28] carbondata git commit: [CARBONDATA-1720][FILTER] Wrong data
displayed for <= filter for timestamp column(dictionary column)
Posted by ja...@apache.org.
[CARBONDATA-1720][FILTER] Wrong data displayed for <= filter for timestamp column(dictionary column)
Issue:
<= filter is giving wrong results for timestamp dictioinary column
Solution:
In less than equal to filter, we are considering surrogate 2 as default value. But surrogate 1 is for default value.
This closes #1502
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/17892b17
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/17892b17
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/17892b17
Branch: refs/heads/fgdatamap
Commit: 17892b17b688eaa637b3dd97c25286edb4183eaa
Parents: 09d0205
Author: dhatchayani <dh...@gmail.com>
Authored: Wed Nov 15 18:41:00 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Nov 16 16:12:19 2017 +0530
----------------------------------------------------------------------
.../executer/RangeValueFilterExecuterImpl.java | 2 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 10 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 14 +-
.../src/test/resources/timestamp.csv | 301 +++++++++++++++++++
.../RangeFilterAllDataTypesTestCases.scala | 9 +
5 files changed, 327 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 0cfa198..ee373c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -554,7 +554,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
if (dimColEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType());
- int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+ int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex);
if (currentBlockDimension.isSortColumn()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 5e0bb41..88cf75c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -267,7 +267,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(
dimColEvaluatorInfoList.get(0).getDimension().getDataType());
- int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+ int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
if (currentBlockDimension.isSortColumn()) {
@@ -324,7 +324,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return bitSet;
}
} else {
- skip = start;
+ // as start will be last index of null value inclusive
+ // so adding 1 to skip last null value
+ skip = start + 1;
}
startIndex = skip;
}
@@ -392,7 +394,9 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return bitSet;
}
} else {
- skip = start;
+ // as start will be last index of null value inclusive
+ // so adding 1 to skip last null value
+ skip = start + 1;
}
startIndex = skip;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 74162f6..7f735c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -269,7 +269,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(
dimColEvaluatorInfoList.get(0).getDimension().getDataType());
- int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+ int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
if (currentBlockDimension.isSortColumn()) {
@@ -319,7 +319,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
if (null != defaultValue) {
start = CarbonUtil
.getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- defaultValue, false);
+ defaultValue, true);
if (start < 0) {
skip = -(start + 1);
// end of block
@@ -327,7 +327,9 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return bitSet;
}
} else {
- skip = start;
+ // as start will be last index of null value inclusive
+ // so adding 1 to skip last null value
+ skip = start + 1;
}
startIndex = skip;
}
@@ -395,7 +397,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
if (null != defaultValue) {
start = CarbonUtil
.getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
- numerOfRows - 1, defaultValue, false);
+ numerOfRows - 1, defaultValue, true);
if (start < 0) {
skip = -(start + 1);
// end of block
@@ -403,7 +405,9 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return bitSet;
}
} else {
- skip = start;
+ // as start will be last index of null value inclusive
+ // so adding 1 to skip last null value
+ skip = start + 1;
}
startIndex = skip;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/integration/spark-common-test/src/test/resources/timestamp.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/timestamp.csv b/integration/spark-common-test/src/test/resources/timestamp.csv
new file mode 100644
index 0000000..0a7b3ff
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/timestamp.csv
@@ -0,0 +1,301 @@
+timestampCol
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
+1970-01-01 05:30:00
http://git-wip-us.apache.org/repos/asf/carbondata/blob/17892b17/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
index 00be125..87cc194 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
@@ -263,6 +263,15 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test range filter for less than filter"){
+ sql("drop table if exists timestampTable")
+ sql("create table timestampTable (timestampCol timestamp) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='timestampCol')")
+ sql(s"load data local inpath '$resourcesPath/timestamp.csv' into table timestampTable")
+ checkAnswer(sql("select * from timestampTable where timestampCol='1970-01-01 05:30:00'"),
+ sql("select * from timestampTable where timestampCol<='1970-01-01 05:30:00'"))
+ sql("drop table if exists timestampTable")
+ }
+
test("test range filter for direct dictionary not equality"){
checkAnswer(
sql("select doj from directDictionaryTable where doj != '2016-03-14 15:00:16'"),
[12/28] carbondata git commit: [CARBONDATA-1326] Fixed high priority
findbug issue
Posted by ja...@apache.org.
[CARBONDATA-1326] Fixed high priority findbug issue
Fixed high priority findbug issue
This closes #1507
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0f46ef04
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0f46ef04
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0f46ef04
Branch: refs/heads/fgdatamap
Commit: 0f46ef04d66a513f0987b05ace393016c151fd1c
Parents: 5fc7f06
Author: dhatchayani <dh...@gmail.com>
Authored: Thu Nov 16 16:52:18 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Nov 17 14:51:58 2017 +0800
----------------------------------------------------------------------
.../core/cache/dictionary/ManageDictionaryAndBTree.java | 12 +++++-------
.../datastore/page/encoding/bool/BooleanConvert.java | 4 +++-
.../core/statusmanager/SegmentStatusManager.java | 8 ++++++--
.../org/apache/carbondata/core/util/CarbonUtil.java | 3 ++-
4 files changed, 16 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0f46ef04/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
index f8d2495..4f8f724 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
@@ -102,14 +102,12 @@ public class ManageDictionaryAndBTree {
// clear Btree cache from LRU cache
LoadMetadataDetails[] loadMetadataDetails =
SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath());
- if (null != loadMetadataDetails) {
- String[] segments = new String[loadMetadataDetails.length];
- int i = 0;
- for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
- segments[i++] = loadMetadataDetail.getLoadName();
- }
- invalidateBTreeCache(carbonTable.getAbsoluteTableIdentifier(), segments);
+ String[] segments = new String[loadMetadataDetails.length];
+ int i = 0;
+ for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
+ segments[i++] = loadMetadataDetail.getLoadName();
}
+ invalidateBTreeCache(carbonTable.getAbsoluteTableIdentifier(), segments);
// clear dictionary cache from LRU cache
List<CarbonDimension> dimensions =
carbonTable.getDimensionByTableName(carbonTable.getTableName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0f46ef04/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
index b373adf..10a9767 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
@@ -17,6 +17,8 @@
package org.apache.carbondata.core.datastore.page.encoding.bool;
+import java.util.Locale;
+
/**
* convert tools for boolean data type
*/
@@ -51,7 +53,7 @@ public class BooleanConvert {
* @return Boolean type data
*/
public static Boolean parseBoolean(String input) {
- String value = input.toLowerCase();
+ String value = input.toLowerCase(Locale.getDefault());
if (("false").equals(value)) {
return Boolean.FALSE;
} else if (("true").equals(value)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0f46ef04/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index e3dbfed..1944f96 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -120,6 +120,10 @@ public class SegmentStatusManager {
BufferedReader buffReader =
new BufferedReader(new InputStreamReader(dataInputStream, "UTF-8"));
loadFolderDetailsArray = gson.fromJson(buffReader, LoadMetadataDetails[].class);
+ // if loadFolderDetailsArray is null, assign a empty array
+ if (null == loadFolderDetailsArray) {
+ loadFolderDetailsArray = new LoadMetadataDetails[0];
+ }
//just directly iterate Array
for (LoadMetadataDetails segment : loadFolderDetailsArray) {
if (SegmentStatus.SUCCESS == segment.getSegmentStatus() ||
@@ -287,7 +291,7 @@ public class SegmentStatusManager {
}
// read existing metadata details in load metadata.
listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath);
- if (listOfLoadFolderDetailsArray != null && listOfLoadFolderDetailsArray.length != 0) {
+ if (listOfLoadFolderDetailsArray.length != 0) {
updateDeletionStatus(loadIds, listOfLoadFolderDetailsArray, invalidLoadIds);
if (invalidLoadIds.isEmpty()) {
// All or None , if anything fails then dont write
@@ -371,7 +375,7 @@ public class SegmentStatusManager {
}
// read existing metadata details in load metadata.
listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath);
- if (listOfLoadFolderDetailsArray != null && listOfLoadFolderDetailsArray.length != 0) {
+ if (listOfLoadFolderDetailsArray.length != 0) {
updateDeletionStatus(loadDate, listOfLoadFolderDetailsArray, invalidLoadTimestamps,
loadStartTime);
if (invalidLoadTimestamps.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0f46ef04/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index bdd7ba3..1e402d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -35,6 +35,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
@@ -753,7 +754,7 @@ public final class CarbonUtil {
}
private static boolean checkIfPrefixExists(String path) {
- final String lowerPath = path.toLowerCase();
+ final String lowerPath = path.toLowerCase(Locale.getDefault());
return lowerPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX) ||
lowerPath.startsWith(CarbonCommonConstants.VIEWFSURL_PREFIX) ||
lowerPath.startsWith(CarbonCommonConstants.LOCAL_FILE_PREFIX) ||
[22/28] carbondata git commit: [CARBONDATA-1614][Streaming] Show file
format for segment
Posted by ja...@apache.org.
[CARBONDATA-1614][Streaming] Show file format for segment
This closes #1498
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ee71610e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ee71610e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ee71610e
Branch: refs/heads/fgdatamap
Commit: ee71610e1c7686117f3feebab75fdeb82dc31d54
Parents: 91355ef
Author: Jacky Li <ja...@qq.com>
Authored: Wed Nov 15 00:05:03 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Nov 18 00:28:25 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/statusmanager/FileFormat.java | 17 +++++++++++------
.../core/statusmanager/LoadMetadataDetails.java | 2 +-
.../apache/carbondata/hadoop/CarbonInputSplit.java | 2 +-
.../carbondata/hadoop/CarbonMultiBlockSplit.java | 2 +-
.../hadoop/api/CarbonTableInputFormat.java | 6 +++---
.../streaming/CarbonStreamInputFormatTest.java | 2 +-
.../org/apache/carbondata/api/CarbonStore.scala | 3 ++-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 12 ++++++------
.../carbondata/spark/rdd/CarbonScanRDD.scala | 6 +++---
.../apache/carbondata/spark/util/CommonUtil.scala | 4 ++--
.../apache/spark/sql/CarbonCatalystOperators.scala | 1 +
.../segmentreading/TestSegmentReading.scala | 2 +-
.../carbondata/TestStreamingTableOperation.scala | 7 ++++---
.../streaming/segment/StreamSegment.java | 6 +++---
14 files changed, 40 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormat.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormat.java
index 83a4813..c154c5f 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormat.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormat.java
@@ -18,23 +18,28 @@
package org.apache.carbondata.core.statusmanager;
/**
- * the data file format which was supported
+ * The data file format supported in carbondata project
*/
public enum FileFormat {
- carbondata, rowformat;
+
+ // carbondata columnar file format, optimized for read
+ COLUMNAR_V3,
+
+ // carbondata row file format, optimized for write
+ ROW_V1;
public static FileFormat getByOrdinal(int ordinal) {
if (ordinal < 0 || ordinal >= FileFormat.values().length) {
- return carbondata;
+ return COLUMNAR_V3;
}
switch (ordinal) {
case 0:
- return carbondata;
+ return COLUMNAR_V3;
case 1:
- return rowformat;
+ return ROW_V1;
}
- return carbondata;
+ return COLUMNAR_V3;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index b282d53..bb7fc9d 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -98,7 +98,7 @@ public class LoadMetadataDetails implements Serializable {
/**
* the file format of this segment
*/
- private FileFormat fileFormat = FileFormat.carbondata;
+ private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
public String getPartitionCount() {
return partitionCount;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index f7b372f..e89c2d6 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -82,7 +82,7 @@ public class CarbonInputSplit extends FileSplit
private BlockletDetailInfo detailInfo;
- private FileFormat fileFormat = FileFormat.carbondata;
+ private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
public CarbonInputSplit() {
segmentId = null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
index d3fa2c2..96fe909 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
@@ -45,7 +45,7 @@ public class CarbonMultiBlockSplit extends InputSplit implements Writable {
*/
private String[] locations;
- private FileFormat fileFormat = FileFormat.carbondata;
+ private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
public CarbonMultiBlockSplit() {
splitList = null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 552455a..8bf779e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -475,19 +475,19 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
splits.add(makeSplit(segmentId, path, length - bytesRemaining, splitSize,
blkLocations[blkIndex].getHosts(),
- blkLocations[blkIndex].getCachedHosts(), FileFormat.rowformat));
+ blkLocations[blkIndex].getCachedHosts(), FileFormat.ROW_V1));
bytesRemaining -= splitSize;
}
if (bytesRemaining != 0) {
int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
splits.add(makeSplit(segmentId, path, length - bytesRemaining, bytesRemaining,
blkLocations[blkIndex].getHosts(),
- blkLocations[blkIndex].getCachedHosts(), FileFormat.rowformat));
+ blkLocations[blkIndex].getCachedHosts(), FileFormat.ROW_V1));
}
} else {
//Create empty hosts array for zero length files
splits.add(makeSplit(segmentId, path, 0, length, new String[0],
- FileFormat.rowformat));
+ FileFormat.ROW_V1));
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java
index 9970c50..4f81518 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java
@@ -73,7 +73,7 @@ public class CarbonStreamInputFormatTest extends TestCase {
List<CarbonInputSplit> splitList = new ArrayList<>();
splitList.add(carbonInputSplit);
return new CarbonMultiBlockSplit(identifier, splitList, new String[] { "localhost" },
- FileFormat.rowformat);
+ FileFormat.ROW_V1);
}
@Test public void testCreateRecordReader() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index a2c9c6d..6c2490e 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -29,7 +29,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier}
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
@@ -90,6 +90,7 @@ object CarbonStore {
load.getSegmentStatus.getMessage,
startTime,
endTime,
+ load.getFileFormat.toString,
mergedTo)
}.toSeq
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 2a7ca47..aaeedb4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -306,10 +306,10 @@ class CarbonMergerRDD[K, V](
val splits = format.getSplits(job)
// keep on assigning till last one is reached.
- if (null != splits && splits.size > 0) {
- splitsOfLastSegment = splits.asScala.map(_.asInstanceOf[CarbonInputSplit])
- .filter { split => FileFormat.carbondata.equals(split.getFileFormat) }.toList.asJava
- }
+ if (null != splits && splits.size > 0) splitsOfLastSegment =
+ splits.asScala
+ .map(_.asInstanceOf[CarbonInputSplit])
+ .filter { split => FileFormat.COLUMNAR_V3.equals(split.getFileFormat) }.toList.asJava
carbonInputSplits ++:= splits.asScala.map(_.asInstanceOf[CarbonInputSplit]).filter(entry => {
val blockInfo = new TableBlockInfo(entry.getPath.toString,
@@ -317,10 +317,10 @@ class CarbonMergerRDD[K, V](
entry.getLocations, entry.getLength, entry.getVersion,
updateStatusManager.getDeleteDeltaFilePath(entry.getPath.toString)
)
- (((!updated) || ((updated) && (!CarbonUtil
+ ((!updated) || (updated && (!CarbonUtil
.isInvalidTableBlock(blockInfo.getSegmentId, blockInfo.getFilePath,
updateDetails, updateStatusManager)))) &&
- FileFormat.carbondata.equals(entry.getFileFormat))
+ FileFormat.COLUMNAR_V3.equals(entry.getFileFormat)
})
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index a84b040..b24562c 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -94,7 +94,7 @@ class CarbonScanRDD(
val streamSplits = new ArrayBuffer[InputSplit]()
splits.asScala.foreach { split =>
val carbonInputSplit = split.asInstanceOf[CarbonInputSplit]
- if (FileFormat.rowformat == carbonInputSplit.getFileFormat) {
+ if (FileFormat.ROW_V1 == carbonInputSplit.getFileFormat) {
streamSplits += split
} else {
columnarSplits.add(split)
@@ -111,7 +111,7 @@ class CarbonScanRDD(
new CarbonMultiBlockSplit(identifier,
Seq(splitWithIndex._1.asInstanceOf[CarbonInputSplit]).asJava,
splitWithIndex._1.getLocations,
- FileFormat.rowformat)
+ FileFormat.ROW_V1)
new CarbonSparkPartition(id, splitWithIndex._2 + index, multiBlockSplit)
}
if (batchPartitions.isEmpty) {
@@ -250,7 +250,7 @@ class CarbonScanRDD(
val model = format.getQueryModel(inputSplit, attemptContext)
// get RecordReader by FileFormat
val reader: RecordReader[Void, Object] = inputSplit.getFileFormat match {
- case FileFormat.rowformat =>
+ case FileFormat.ROW_V1 =>
// create record reader for row format
DataTypeUtil.setDataTypeConverter(new SparkDataTypeConverterImpl)
val inputFormat = new CarbonStreamInputFormat
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index f0b33f4..a922a07 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -859,14 +859,14 @@ object CommonUtil {
CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT,
CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT).toBoolean) {
new CarbonMergeFilesRDD(sparkContext, AbsoluteTableIdentifier.from(tablePath,
- carbonTable.getDatabaseName, carbonTable.getFactTableName).getTablePath,
+ carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
segmentIds).collect()
}
} catch {
case _: Exception =>
if (CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT.toBoolean) {
new CarbonMergeFilesRDD(sparkContext, AbsoluteTableIdentifier.from(tablePath,
- carbonTable.getDatabaseName, carbonTable.getFactTableName).getTablePath,
+ carbonTable.getDatabaseName, carbonTable.getTableName).getTablePath,
segmentIds).collect()
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
index 62632df..f8a5404 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
@@ -123,6 +123,7 @@ case class ShowLoadsCommand(
AttributeReference("Status", StringType, nullable = false)(),
AttributeReference("Load Start Time", TimestampType, nullable = false)(),
AttributeReference("Load End Time", TimestampType, nullable = true)(),
+ AttributeReference("File Format", StringType, nullable = false)(),
AttributeReference("Merged To", StringType, nullable = false)())
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
index ac3fa5c..b23ba2c 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
@@ -232,7 +232,7 @@ class TestSegmentReading extends QueryTest with BeforeAndAfterAll {
|('DELIMITER'= ',', 'QUOTECHAR'= '\"')""".stripMargin)
val df = sql("SHOW SEGMENTS for table carbon_table_show_seg")
val col = df.collect().map{
- row => Row(row.getString(0),row.getString(1),row.getString(4))
+ row => Row(row.getString(0),row.getString(1),row.getString(5))
}.toSeq
assert(col.equals(Seq(Row("2","Success","NA"),
Row("1","Compacted","0.1"),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index b29cca4..33aa2c9 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -32,8 +32,7 @@ import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.sql.types.StructType
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.statusmanager.SegmentStatus
+import org.apache.carbondata.core.statusmanager.{FileFormat, SegmentStatus}
import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.hadoop.streaming.CarbonStreamOutputFormat
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
@@ -561,11 +560,13 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
}
sql("alter table streaming.stream_table_compact compact 'minor'")
+ sql("show segments for table streaming.stream_table_compact").show
val result = sql("show segments for table streaming.stream_table_compact").collect()
result.foreach { row =>
if (row.getString(0).equals("1")) {
assertResult(SegmentStatus.STREAMING.getMessage)(row.getString(1))
+ assertResult(FileFormat.ROW_V1.toString)(row.getString(4))
}
}
}
@@ -583,7 +584,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
badRecordAction = "force",
handoffSize = 1024L * 200
)
- assert(sql("show segments for table streaming.stream_table_new").count() == 4)
+ assert(sql("show segments for table streaming.stream_table_new").count() > 1)
checkAnswer(
sql("select count(*) from streaming.stream_table_new"),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee71610e/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
index 7682437..0187597 100644
--- a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
+++ b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
@@ -73,7 +73,7 @@ public class StreamSegment {
SegmentStatusManager.readLoadMetadata(tablePath.getMetadataDirectoryPath());
LoadMetadataDetails streamSegment = null;
for (LoadMetadataDetails detail : details) {
- if (FileFormat.rowformat == detail.getFileFormat()) {
+ if (FileFormat.ROW_V1 == detail.getFileFormat()) {
if (SegmentStatus.STREAMING == detail.getSegmentStatus()) {
streamSegment = detail;
break;
@@ -85,7 +85,7 @@ public class StreamSegment {
LoadMetadataDetails newDetail = new LoadMetadataDetails();
newDetail.setPartitionCount("0");
newDetail.setLoadName("" + segmentId);
- newDetail.setFileFormat(FileFormat.rowformat);
+ newDetail.setFileFormat(FileFormat.ROW_V1);
newDetail.setLoadStartTime(System.currentTimeMillis());
newDetail.setSegmentStatus(SegmentStatus.STREAMING);
@@ -149,7 +149,7 @@ public class StreamSegment {
LoadMetadataDetails newDetail = new LoadMetadataDetails();
newDetail.setPartitionCount("0");
newDetail.setLoadName("" + newSegmentId);
- newDetail.setFileFormat(FileFormat.rowformat);
+ newDetail.setFileFormat(FileFormat.ROW_V1);
newDetail.setLoadStartTime(System.currentTimeMillis());
newDetail.setSegmentStatus(SegmentStatus.STREAMING);