You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/02/05 15:02:19 UTC
[01/50] [abbrv] carbondata git commit: [CARBONDATA-2095]Copy data of
GenericInternalRow for each iteration when converting stream segment to batch
segment [Forced Update!]
Repository: carbondata
Updated Branches:
refs/heads/fgdatamap af8ba6d64 -> e972fd3d5 (forced update)
[CARBONDATA-2095]Copy data of GenericInternalRow for each iteration when converting stream segment to batch segment
This closes #1881
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2081fba7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2081fba7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2081fba7
Branch: refs/heads/fgdatamap
Commit: 2081fba7f845c57fe4332bc8f33680a91c842720
Parents: 9ec64a6
Author: Zhang Zhichao <44...@qq.com>
Authored: Tue Jan 30 16:52:46 2018 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Tue Jan 30 18:39:40 2018 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/streaming/StreamHandoffRDD.scala | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2081fba7/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index d092580..a96ab32 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -75,11 +75,14 @@ class StreamingRawResultIterator(
}
override def next(): Array[Object] = {
- recordReader
+ val rowTmp = recordReader
.getCurrentValue
.asInstanceOf[GenericInternalRow]
.values
.asInstanceOf[Array[Object]]
+ val row = new Array[Object](rowTmp.length)
+ System.arraycopy(rowTmp, 0, row, 0, rowTmp.length)
+ row
}
}
[15/50] [abbrv] carbondata git commit: [HOTFIX]modify default config:
change the default of tempCSV to false
Posted by ra...@apache.org.
[HOTFIX]modify default config: change the default of tempCSV to false
This closes #1806
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/099a0473
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/099a0473
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/099a0473
Branch: refs/heads/fgdatamap
Commit: 099a047389e5f6d30851d18c2a0b03d31c3fcf1c
Parents: 43a37d7
Author: qiuchenjian <80...@qq.com>
Authored: Mon Jan 15 22:47:24 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 16:46:43 2018 +0800
----------------------------------------------------------------------
.../src/main/scala/org/apache/carbondata/spark/CarbonOption.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/099a0473/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
index bcdad26..dfd9567 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
@@ -36,7 +36,7 @@ class CarbonOption(options: Map[String, String]) {
"org.apache.carbondata.processing.partition.impl.SampleDataPartitionerImpl")
}
- def tempCSV: Boolean = options.getOrElse("tempCSV", "true").toBoolean
+ def tempCSV: Boolean = options.getOrElse("tempCSV", "false").toBoolean
def compress: Boolean = options.getOrElse("compress", "false").toBoolean
[47/50] [abbrv] carbondata git commit: [CARBONDATA-2080]
[S3-Implementation] Propagated hadoopConf from driver to executor for s3
implementation in cluster mode.
Posted by ra...@apache.org.
[CARBONDATA-2080] [S3-Implementation] Propagated hadoopConf from driver to executor for s3 implementation in cluster mode.
Problem : hadoopconf was not getting propagated from driver to the executor that's why load was failing to the distributed environment.
Solution: Setting the Hadoop conf in base class CarbonRDD
How to verify this PR :
Execute the load in the cluster mode It should be a success using location s3.
This closes #1860
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e502c59a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e502c59a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e502c59a
Branch: refs/heads/fgdatamap
Commit: e502c59a2d0b95d80db3aff04c749654254eadbe
Parents: c3e9968
Author: Jatin <ja...@knoldus.in>
Authored: Thu Jan 25 16:53:00 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Feb 3 20:17:10 2018 +0800
----------------------------------------------------------------------
.../spark/rdd/AlterTableAddColumnRDD.scala | 2 +-
.../spark/rdd/AlterTableDropColumnRDD.scala | 2 +-
.../spark/rdd/CarbonCleanFilesRDD.scala | 2 +-
.../spark/rdd/CarbonDeleteLoadByDateRDD.scala | 2 +-
.../spark/rdd/CarbonDeleteLoadRDD.scala | 2 +-
.../spark/rdd/CarbonDropPartitionRDD.scala | 4 +--
.../spark/rdd/CarbonDropTableRDD.scala | 2 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 3 +-
.../spark/rdd/CarbonMergeFilesRDD.scala | 2 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +-
.../apache/carbondata/spark/rdd/CarbonRDD.scala | 32 ++++++++++++++++++--
.../spark/rdd/NewCarbonDataLoadRDD.scala | 2 +-
.../carbondata/spark/rdd/SparkDataMapJob.scala | 2 +-
.../apache/spark/rdd/DataLoadCoalescedRDD.scala | 3 +-
.../apache/spark/rdd/UpdateCoalescedRDD.scala | 2 +-
.../carbondata/streaming/StreamHandoffRDD.scala | 2 +-
16 files changed, 47 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
index 56a66b9..7c1edea 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
@@ -50,7 +50,7 @@ class AddColumnPartition(rddId: Int, idx: Int, schema: ColumnSchema) extends Par
class AlterTableAddColumnRDD[K, V](sc: SparkContext,
@transient newColumns: Seq[ColumnSchema],
identifier: AbsoluteTableIdentifier)
- extends CarbonRDD[(Int, SegmentStatus)](sc, Nil) {
+ extends CarbonRDD[(Int, SegmentStatus)](sc, Nil, sc.hadoopConfiguration) {
val lockType: String = CarbonProperties.getInstance.getProperty(CarbonCommonConstants.LOCK_TYPE,
CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
index 248f351..e14524e 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableDropColumnRDD.scala
@@ -48,7 +48,7 @@ class DropColumnPartition(rddId: Int, idx: Int, schema: ColumnSchema) extends Pa
class AlterTableDropColumnRDD[K, V](sc: SparkContext,
@transient newColumns: Seq[ColumnSchema],
carbonTableIdentifier: AbsoluteTableIdentifier)
- extends CarbonRDD[(Int, SegmentStatus)](sc, Nil) {
+ extends CarbonRDD[(Int, SegmentStatus)](sc, Nil, sc.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
newColumns.zipWithIndex.map { column =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
index 32523d8..9936a2a 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonCleanFilesRDD.scala
@@ -33,7 +33,7 @@ class CarbonCleanFilesRDD[V: ClassTag](
databaseName: String,
tableName: String,
partitioner: Partitioner)
- extends CarbonRDD[V](sc, Nil) {
+ extends CarbonRDD[V](sc, Nil, sc.hadoopConfiguration) {
sc.setLocalProperty("spark.scheduler.pool", "DDL")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
index 45271a7..b11dfad 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
@@ -39,7 +39,7 @@ class CarbonDeleteLoadByDateRDD[K, V](
dimTableName: String,
storePath: String,
loadMetadataDetails: List[LoadMetadataDetails])
- extends CarbonRDD[(K, V)](sc, Nil) {
+ extends CarbonRDD[(K, V)](sc, Nil, sc.hadoopConfiguration) {
sc.setLocalProperty("spark.scheduler.pool", "DDL")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadRDD.scala
index 9a1ef33..759ed42 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadRDD.scala
@@ -34,7 +34,7 @@ class CarbonDeleteLoadRDD[V: ClassTag](
databaseName: String,
tableName: String,
partitioner: Partitioner)
- extends CarbonRDD[V](sc, Nil) {
+ extends CarbonRDD[V](sc, Nil, sc.hadoopConfiguration) {
sc.setLocalProperty("spark.scheduler.pool", "DDL")
override def getPartitions: Array[Partition] = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
index 4806f9f..800cc36 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
@@ -47,7 +47,7 @@ class CarbonDropPartitionRDD(
partitions: Seq[String],
uniqueId: String,
partialMatch: Boolean)
- extends CarbonRDD[String](sc, Nil) {
+ extends CarbonRDD[String](sc, Nil, sc.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
segments.zipWithIndex.map {s =>
@@ -105,7 +105,7 @@ class CarbonDropPartitionCommitRDD(
success: Boolean,
uniqueId: String,
partitions: Seq[String])
- extends CarbonRDD[String](sc, Nil) {
+ extends CarbonRDD[String](sc, Nil, sc.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
segments.zipWithIndex.map {s =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropTableRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropTableRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropTableRDD.scala
index 652720c..f327d88 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropTableRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropTableRDD.scala
@@ -30,7 +30,7 @@ class CarbonDropTableRDD[V: ClassTag](
valueClass: Value[V],
databaseName: String,
tableName: String)
- extends CarbonRDD[V](sc, Nil) {
+ extends CarbonRDD[V](sc, Nil, sc.hadoopConfiguration) {
sc.setLocalProperty("spark.scheduler.pool", "DDL")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 7acf4e2..cf22b3d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -506,7 +506,8 @@ class CarbonColumnDictGenerateRDD(carbonLoadModel: CarbonLoadModel,
table: CarbonTableIdentifier,
dimensions: Array[CarbonDimension],
dictFolderPath: String)
- extends CarbonRDD[(Int, ColumnDistinctValues)](sparkContext, Nil) {
+ extends CarbonRDD[(Int, ColumnDistinctValues)](sparkContext, Nil,
+ sparkContext.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
val primDimensions = dictionaryLoadModel.primDimensions
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergeFilesRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergeFilesRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergeFilesRDD.scala
index 1087ea7..3f38300 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergeFilesRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergeFilesRDD.scala
@@ -41,7 +41,7 @@ class CarbonMergeFilesRDD(
tablePath: String,
segments: Seq[String],
readFileFooterFromCarbonDataFile: Boolean)
- extends CarbonRDD[String](sc, Nil) {
+ extends CarbonRDD[String](sc, Nil, sc.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
segments.zipWithIndex.map {s =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 0d0f024..fa126fc 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -63,7 +63,7 @@ class CarbonMergerRDD[K, V](
carbonLoadModel: CarbonLoadModel,
carbonMergerMapping: CarbonMergerMapping,
confExecutorsTemp: String)
- extends CarbonRDD[(K, V)](sc, Nil) {
+ extends CarbonRDD[(K, V)](sc, Nil, sc.hadoopConfiguration) {
sc.setLocalProperty("spark.scheduler.pool", "DDL")
sc.setLocalProperty("spark.job.interruptOnCancel", "true")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
index bf46f67..6f248d2 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
@@ -17,20 +17,26 @@
package org.apache.carbondata.spark.rdd
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream}
+
import scala.collection.JavaConverters._
import scala.reflect.ClassTag
+import org.apache.hadoop.conf.Configuration
import org.apache.spark.{Dependency, OneToOneDependency, Partition, SparkContext, TaskContext}
import org.apache.spark.rdd.RDD
+import org.apache.carbondata.core.datastore.compression.CompressorFactory
import org.apache.carbondata.core.metadata.schema.table.TableInfo
import org.apache.carbondata.core.util._
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
/**
* This RDD maintains session level ThreadLocal
*/
abstract class CarbonRDD[T: ClassTag](@transient sc: SparkContext,
- @transient private var deps: Seq[Dependency[_]]) extends RDD[T](sc, deps) {
+ @transient private var deps: Seq[Dependency[_]],
+ @transient hadoopConf: Configuration) extends RDD[T](sc, deps) {
val carbonSessionInfo: CarbonSessionInfo = {
var info = ThreadLocalSessionInfo.getCarbonSessionInfo
@@ -42,14 +48,24 @@ abstract class CarbonRDD[T: ClassTag](@transient sc: SparkContext,
info
}
+ private val confBytes = {
+ val bao = new ByteArrayOutputStream()
+ val oos = new ObjectOutputStream(bao)
+ hadoopConf.write(oos)
+ oos.close()
+ CompressorFactory.getInstance().getCompressor.compressByte(bao.toByteArray)
+ }
+
/** Construct an RDD with just a one-to-one dependency on one parent */
def this(@transient oneParent: RDD[_]) =
- this (oneParent.context, List(new OneToOneDependency(oneParent)))
+ this (oneParent.context, List(new OneToOneDependency(oneParent)),
+ oneParent.sparkContext.hadoopConfiguration)
// RDD compute logic should be here
def internalCompute(split: Partition, context: TaskContext): Iterator[T]
final def compute(split: Partition, context: TaskContext): Iterator[T] = {
+ CarbonInputFormatUtil.setS3Configurations(getConf)
ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)
TaskMetricsMap.threadLocal.set(Thread.currentThread().getId)
val carbonTaskInfo = new CarbonTaskInfo
@@ -59,6 +75,16 @@ abstract class CarbonRDD[T: ClassTag](@transient sc: SparkContext,
map(f => CarbonProperties.getInstance().addProperty(f._1, f._2))
internalCompute(split, context)
}
+
+ private def getConf: Configuration = {
+ val configuration = new Configuration(false)
+ val bai = new ByteArrayInputStream(CompressorFactory.getInstance().getCompressor
+ .unCompressByte(confBytes))
+ val ois = new ObjectInputStream(bai)
+ configuration.readFields(ois)
+ ois.close()
+ configuration
+ }
}
/**
@@ -67,7 +93,7 @@ abstract class CarbonRDD[T: ClassTag](@transient sc: SparkContext,
abstract class CarbonRDDWithTableInfo[T: ClassTag](
@transient sc: SparkContext,
@transient private var deps: Seq[Dependency[_]],
- serializedTableInfo: Array[Byte]) extends CarbonRDD[T](sc, deps) {
+ serializedTableInfo: Array[Byte]) extends CarbonRDD[T](sc, deps, sc.hadoopConfiguration) {
def this(@transient oneParent: RDD[_], serializedTableInfo: Array[Byte]) =
this (oneParent.context, List(new OneToOneDependency(oneParent)), serializedTableInfo)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index e17824f..06acbba 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -182,7 +182,7 @@ class NewCarbonDataLoadRDD[K, V](
carbonLoadModel: CarbonLoadModel,
blocksGroupBy: Array[(String, Array[BlockDetails])],
@transient hadoopConf: Configuration)
- extends CarbonRDD[(K, V)](sc, Nil) {
+ extends CarbonRDD[(K, V)](sc, Nil, hadoopConf) {
sc.setLocalProperty("spark.scheduler.pool", "DDL")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala
index 600cd80..60052f0 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala
@@ -58,7 +58,7 @@ class DataMapRDDPartition(rddId: Int, idx: Int, val inputSplit: InputSplit) exte
class DataMapPruneRDD(sc: SparkContext,
dataMapFormat: DistributableDataMapFormat,
resolverIntf: FilterResolverIntf)
- extends CarbonRDD[(ExtendedBlocklet)](sc, Nil) {
+ extends CarbonRDD[(ExtendedBlocklet)](sc, Nil, sc.hadoopConfiguration) {
private val jobTrackerId: String = {
val formatter = new SimpleDateFormat("yyyyMMddHHmm")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/spark/rdd/DataLoadCoalescedRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/rdd/DataLoadCoalescedRDD.scala b/integration/spark-common/src/main/scala/org/apache/spark/rdd/DataLoadCoalescedRDD.scala
index 2157799..6a97477 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/rdd/DataLoadCoalescedRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/rdd/DataLoadCoalescedRDD.scala
@@ -29,7 +29,8 @@ case class DataLoadPartitionWrap[T: ClassTag](rdd: RDD[T], partition: Partition)
class DataLoadCoalescedRDD[T: ClassTag](
@transient var prev: RDD[T],
nodeList: Array[String])
- extends CarbonRDD[DataLoadPartitionWrap[T]](prev.context, Nil) {
+ extends CarbonRDD[DataLoadPartitionWrap[T]](prev.context, Nil,
+ prev.sparkContext.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
new DataLoadPartitionCoalescer(prev, nodeList).run
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/integration/spark-common/src/main/scala/org/apache/spark/rdd/UpdateCoalescedRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/rdd/UpdateCoalescedRDD.scala b/integration/spark-common/src/main/scala/org/apache/spark/rdd/UpdateCoalescedRDD.scala
index 9befcaa..bcca7ed 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/rdd/UpdateCoalescedRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/rdd/UpdateCoalescedRDD.scala
@@ -29,7 +29,7 @@ import org.apache.carbondata.spark.rdd.CarbonRDD
class UpdateCoalescedRDD[T: ClassTag](
@transient var prev: RDD[T],
nodeList: Array[String])
- extends CarbonRDD[T](prev.context, Nil) {
+ extends CarbonRDD[T](prev.context, Nil, prev.sparkContext.hadoopConfiguration) {
override def getPartitions: Array[Partition] = {
new DataLoadPartitionCoalescer(prev, nodeList).run
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e502c59a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 186d100..1ecd6e4 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -94,7 +94,7 @@ class StreamHandoffRDD[K, V](
result: HandoffResult[K, V],
carbonLoadModel: CarbonLoadModel,
handOffSegmentId: String
-) extends CarbonRDD[(K, V)](sc, Nil) {
+) extends CarbonRDD[(K, V)](sc, Nil, sc.hadoopConfiguration) {
private val jobTrackerId: String = {
val formatter = new SimpleDateFormat("yyyyMMddHHmm")
[42/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
[CARBONDATA-2099] Refactor query scan process to improve readability
Unified concepts in scan process flow:
1.QueryModel contains all parameter for scan, it is created by API in CarbonTable. (In future, CarbonTable will be the entry point for various table operations)
2.Use term ColumnChunk to represent one column in one blocklet, and use ChunkIndex in reader to read specified column chunk
3.Use term ColumnPage to represent one page in one ColumnChunk
4.QueryColumn => ProjectionColumn, indicating it is for projection
This closes #1874
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/15b4e192
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/15b4e192
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/15b4e192
Branch: refs/heads/fgdatamap
Commit: 15b4e192ee904a2e7c845ac67e0fcf1ba151a683
Parents: 71c2d8c
Author: Jacky Li <ja...@qq.com>
Authored: Tue Jan 30 21:24:04 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Feb 1 10:15:05 2018 +0800
----------------------------------------------------------------------
.../dictionary/AbstractDictionaryCache.java | 3 +-
.../cache/dictionary/DictionaryCacheLoader.java | 7 +-
.../dictionary/DictionaryCacheLoaderImpl.java | 11 +-
.../core/datastore/BTreeBuilderInfo.java | 6 -
.../carbondata/core/datastore/DataRefNode.java | 81 +--
.../carbondata/core/datastore/FileHolder.java | 118 ----
.../carbondata/core/datastore/FileReader.java | 114 +++
.../core/datastore/block/SegmentProperties.java | 50 +-
.../chunk/DimensionColumnDataChunk.java | 116 ---
.../datastore/chunk/DimensionColumnPage.java | 111 +++
.../chunk/impl/AbstractDimensionColumnPage.java | 89 +++
.../chunk/impl/AbstractDimensionDataChunk.java | 95 ---
.../impl/ColumnGroupDimensionColumnPage.java | 194 ++++++
.../impl/ColumnGroupDimensionDataChunk.java | 194 ------
.../chunk/impl/DimensionRawColumnChunk.java | 46 +-
.../impl/FixedLengthDimensionColumnPage.java | 163 +++++
.../impl/FixedLengthDimensionDataChunk.java | 163 -----
.../chunk/impl/MeasureRawColumnChunk.java | 26 +-
.../impl/VariableLengthDimensionColumnPage.java | 133 ++++
.../impl/VariableLengthDimensionDataChunk.java | 140 ----
.../reader/DimensionColumnChunkReader.java | 14 +-
.../chunk/reader/MeasureColumnChunkReader.java | 12 +-
.../AbstractChunkReaderV2V3Format.java | 34 +-
...mpressedDimensionChunkFileBasedReaderV1.java | 38 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 30 +-
...essedDimChunkFileBasedPageLevelReaderV3.java | 11 +-
...mpressedDimensionChunkFileBasedReaderV3.java | 49 +-
.../AbstractMeasureChunkReaderV2V3Format.java | 42 +-
...CompressedMeasureChunkFileBasedReaderV1.java | 16 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 24 +-
...CompressedMeasureChunkFileBasedReaderV3.java | 45 +-
...essedMsrChunkFileBasedPageLevelReaderV3.java | 8 +-
.../chunk/store/ColumnPageWrapper.java | 30 +-
.../chunk/store/DimensionDataChunkStore.java | 8 +-
.../SafeFixedLengthDimensionDataChunkStore.java | 6 +-
...feVariableLengthDimensionDataChunkStore.java | 8 +-
...nsafeFixedLengthDimensionDataChunkStore.java | 10 +-
...afeVariableLengthDimesionDataChunkStore.java | 10 +-
.../datastore/columnar/ColumnGroupModel.java | 26 -
.../core/datastore/impl/DFSFileHolderImpl.java | 166 -----
.../core/datastore/impl/DFSFileReaderImpl.java | 155 ++++
.../datastore/impl/DefaultFileTypeProvider.java | 16 +-
.../core/datastore/impl/FileFactory.java | 4 +-
.../core/datastore/impl/FileHolderImpl.java | 224 ------
.../core/datastore/impl/FileReaderImpl.java | 215 ++++++
.../core/datastore/impl/FileTypeInerface.java | 4 +-
.../impl/btree/AbstractBTreeLeafNode.java | 60 +-
.../impl/btree/BTreeDataRefNodeFinder.java | 6 +-
.../datastore/impl/btree/BTreeNonLeafNode.java | 52 +-
.../impl/btree/BlockBTreeLeafNode.java | 6 +-
.../impl/btree/BlockletBTreeLeafNode.java | 46 +-
.../page/encoding/EncodingFactory.java | 8 +-
.../server/NonSecureDictionaryServer.java | 1 -
.../core/indexstore/BlockletDetailInfo.java | 4 -
.../blockletindex/BlockletDataRefNode.java | 184 +++++
.../BlockletDataRefNodeWrapper.java | 197 ------
.../indexstore/blockletindex/IndexWrapper.java | 2 +-
.../blockletindex/SegmentIndexFileStore.java | 7 +-
.../core/memory/HeapMemoryAllocator.java | 2 +-
.../core/metadata/blocklet/SegmentInfo.java | 19 -
.../core/metadata/schema/table/CarbonTable.java | 130 +++-
.../schema/table/RelationIdentifier.java | 16 -
.../core/metadata/schema/table/TableInfo.java | 6 +-
.../schema/table/column/CarbonColumn.java | 2 +-
.../schema/table/column/CarbonDimension.java | 12 -
.../core/mutate/CarbonUpdateUtil.java | 20 +-
.../core/mutate/DeleteDeltaBlockDetails.java | 8 -
.../core/mutate/DeleteDeltaBlockletDetails.java | 8 -
.../data/BlockletDeleteDeltaCacheLoader.java | 87 ---
.../scan/collector/ScannedResultCollector.java | 16 +-
.../impl/AbstractScannedResultCollector.java | 29 +-
.../impl/DictionaryBasedResultCollector.java | 53 +-
.../DictionaryBasedVectorResultCollector.java | 63 +-
.../collector/impl/RawBasedResultCollector.java | 32 +-
...structureBasedDictionaryResultCollector.java | 17 +-
.../RestructureBasedRawResultCollector.java | 27 +-
.../RestructureBasedVectorResultCollector.java | 33 +-
.../core/scan/complextypes/ArrayQueryType.java | 4 +-
.../scan/complextypes/ComplexQueryType.java | 12 +-
.../scan/complextypes/PrimitiveQueryType.java | 4 +-
.../core/scan/complextypes/StructQueryType.java | 4 +-
.../executor/impl/AbstractQueryExecutor.java | 154 ++--
.../scan/executor/impl/DetailQueryExecutor.java | 6 +-
.../scan/executor/infos/BlockExecutionInfo.java | 179 ++---
.../core/scan/executor/util/QueryUtil.java | 141 ++--
.../scan/executor/util/RestructureUtil.java | 70 +-
.../core/scan/expression/ColumnExpression.java | 8 -
.../scan/expression/FilterModificationNode.java | 35 +-
.../expression/RangeExpressionEvaluator.java | 11 +-
.../scan/filter/FilterExpressionProcessor.java | 22 +-
.../core/scan/filter/FilterProcessor.java | 2 +-
.../carbondata/core/scan/filter/FilterUtil.java | 60 +-
.../core/scan/filter/GenericQueryType.java | 4 +-
.../filter/executer/AndFilterExecuterImpl.java | 22 +-
.../ExcludeColGroupFilterExecuterImpl.java | 85 ---
.../executer/ExcludeFilterExecuterImpl.java | 131 ++--
.../scan/filter/executer/FilterExecuter.java | 10 +-
.../ImplicitIncludeFilterExecutorImpl.java | 23 +-
.../IncludeColGroupFilterExecuterImpl.java | 31 +-
.../executer/IncludeFilterExecuterImpl.java | 142 ++--
.../filter/executer/OrFilterExecuterImpl.java | 25 +-
.../executer/RangeValueFilterExecuterImpl.java | 224 +++---
.../RestructureExcludeFilterExecutorImpl.java | 21 +-
.../RestructureIncludeFilterExecutorImpl.java | 22 +-
.../executer/RowLevelFilterExecuterImpl.java | 151 ++--
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 121 ++--
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 116 +--
...velRangeLessThanEqualFilterExecuterImpl.java | 113 +--
.../RowLevelRangeLessThanFiterExecuterImpl.java | 122 ++--
.../filter/executer/TrueFilterExecutor.java | 17 +-
.../executer/ValueBasedFilterExecuterImpl.java | 16 +-
.../scan/filter/intf/FilterOptimizerBasic.java | 26 -
.../filter/optimizer/RangeFilterOptmizer.java | 3 +-
.../resolver/ConditionalFilterResolverImpl.java | 18 +-
.../resolver/RowLevelFilterResolverImpl.java | 2 +-
.../RowLevelRangeFilterResolverImpl.java | 2 +-
.../MeasureColumnResolvedFilterInfo.java | 12 -
.../TrueConditionalResolverImpl.java | 16 +-
.../visitor/RangeDictionaryColumnVisitor.java | 2 -
.../core/scan/model/CarbonQueryPlan.java | 151 ----
.../core/scan/model/ProjectionColumn.java | 59 ++
.../core/scan/model/ProjectionDimension.java | 45 ++
.../core/scan/model/ProjectionMeasure.java | 46 ++
.../carbondata/core/scan/model/QueryColumn.java | 66 --
.../core/scan/model/QueryDimension.java | 56 --
.../core/scan/model/QueryMeasure.java | 59 --
.../carbondata/core/scan/model/QueryModel.java | 120 +---
.../core/scan/model/QueryProjection.java | 83 +++
.../processor/AbstractDataBlockIterator.java | 251 -------
.../core/scan/processor/BlockletIterator.java | 14 +-
.../core/scan/processor/BlocksChunkHolder.java | 141 ----
.../core/scan/processor/DataBlockIterator.java | 269 +++++++
.../scan/processor/RawBlockletColumnChunks.java | 113 +++
.../processor/impl/DataBlockIteratorImpl.java | 69 --
.../core/scan/result/AbstractScannedResult.java | 698 -------------------
.../core/scan/result/BatchResult.java | 103 ---
.../core/scan/result/BlockletScannedResult.java | 618 ++++++++++++++++
.../carbondata/core/scan/result/RowBatch.java | 103 +++
.../result/impl/FilterQueryScannedResult.java | 50 +-
.../impl/NonFilterQueryScannedResult.java | 14 +-
.../AbstractDetailQueryResultIterator.java | 44 +-
.../scan/result/iterator/ChunkRowIterator.java | 8 +-
.../iterator/DetailQueryResultIterator.java | 14 +-
.../PartitionSpliterRawResultIterator.java | 15 +-
.../scan/result/iterator/RawResultIterator.java | 10 +-
.../VectorDetailQueryResultIterator.java | 4 +-
.../scan/result/vector/CarbonColumnarBatch.java | 3 -
.../scan/result/vector/ColumnVectorInfo.java | 8 +-
.../vector/MeasureDataVectorProcessor.java | 36 +-
.../scan/scanner/AbstractBlockletScanner.java | 181 -----
.../core/scan/scanner/BlockletScanner.java | 22 +-
.../scanner/impl/BlockletFilterScanner.java | 329 +++++++++
.../scan/scanner/impl/BlockletFullScanner.java | 191 +++++
.../core/scan/scanner/impl/FilterScanner.java | 326 ---------
.../scan/scanner/impl/NonFilterScanner.java | 35 -
.../core/scan/wrappers/ByteArrayWrapper.java | 41 +-
.../core/stats/PartitionStatistic.java | 23 -
.../core/stats/QueryStatisticsModel.java | 3 -
.../core/statusmanager/LoadMetadataDetails.java | 9 -
.../statusmanager/SegmentStatusManager.java | 16 -
.../SegmentUpdateStatusManager.java | 156 +----
.../util/AbstractDataFileFooterConverter.java | 1 -
.../apache/carbondata/core/util/CarbonUtil.java | 34 +-
.../core/util/DataFileFooterConverter.java | 6 +-
.../core/writer/CarbonDictionaryWriter.java | 11 -
.../core/writer/CarbonDictionaryWriterImpl.java | 2 +-
.../DictionaryCacheLoaderImplTest.java | 4 +-
.../store/impl/DFSFileHolderImplUnitTest.java | 149 ----
.../store/impl/DFSFileReaderImplUnitTest.java | 149 ++++
.../store/impl/FileHolderImplUnitTest.java | 154 ----
.../store/impl/FileReaderImplUnitTest.java | 154 ++++
.../datastore/block/SegmentPropertiesTest.java | 8 +-
.../impl/ColumnGroupDimensionDataChunkTest.java | 8 +-
.../impl/FixedLengthDimensionDataChunkTest.java | 8 +-
.../impl/btree/BTreeBlockFinderTest.java | 20 +-
.../impl/RawBasedResultCollectorTest.java | 32 +-
.../core/scan/executor/util/QueryUtilTest.java | 41 +-
.../scan/executor/util/RestructureUtilTest.java | 31 +-
.../core/scan/filter/FilterUtilTest.java | 2 +-
.../executer/ExcludeFilterExecuterImplTest.java | 6 +-
.../executer/IncludeFilterExecuterImplTest.java | 32 +-
.../core/scan/result/BatchResultTest.java | 94 ---
.../core/scan/result/RowBatchTest.java | 94 +++
.../carbondata/core/util/CarbonUtilTest.java | 90 +--
.../core/util/DataFileFooterConverterTest.java | 11 +-
.../core/util/RangeFilterProcessorTest.java | 9 +-
.../scanner/impl/FilterScannerTest.java | 160 -----
dev/findbugs-exclude.xml | 2 +-
.../examples/CarbonSessionExample.scala | 2 +-
.../hadoop/api/CarbonTableInputFormat.java | 48 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 4 +-
.../streaming/CarbonStreamRecordReader.java | 2 +-
.../hadoop/util/BlockLevelTraverser.java | 2 +-
.../hadoop/util/CarbonInputFormatUtil.java | 73 +-
.../hive/MapredCarbonInputFormat.java | 9 +-
.../presto/CarbonVectorizedRecordReader.java | 243 -------
.../carbondata/presto/CarbondataPageSource.java | 2 +-
.../presto/CarbondataRecordCursor.java | 6 +-
.../carbondata/presto/CarbondataRecordSet.java | 4 +-
.../presto/CarbondataRecordSetProvider.java | 2 +-
.../PrestoCarbonVectorizedRecordReader.java | 243 +++++++
.../StandardPartitionTableQueryTestCase.scala | 7 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 4 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +-
.../vectorreader/ColumnarVectorWrapper.java | 2 +-
.../VectorizedCarbonRecordReader.java | 51 +-
.../merger/CarbonCompactionExecutor.java | 64 +-
.../partition/impl/QueryPartitionHelper.java | 6 +-
.../spliter/AbstractCarbonQueryExecutor.java | 54 +-
.../partition/spliter/CarbonSplitExecutor.java | 4 +-
.../processing/util/CarbonQueryUtil.java | 6 +-
.../carbondata/streaming/StreamHandoffRDD.scala | 2 +-
212 files changed, 5804 insertions(+), 7359 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index 598d00e..fb67208 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -260,8 +260,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId
DictionaryCacheLoader dictionaryCacheLoader =
new DictionaryCacheLoaderImpl(dictionaryColumnUniqueIdentifier);
dictionaryCacheLoader
- .load(dictionaryInfo, dictionaryColumnUniqueIdentifier.getColumnIdentifier(),
- dictionaryChunkStartOffset, dictionaryChunkEndOffset, loadSortIndex);
+ .load(dictionaryInfo, dictionaryChunkStartOffset, dictionaryChunkEndOffset, loadSortIndex);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
index 01c277d..8df1539 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
@@ -19,8 +19,6 @@ package org.apache.carbondata.core.cache.dictionary;
import java.io.IOException;
-import org.apache.carbondata.core.metadata.ColumnIdentifier;
-
public interface DictionaryCacheLoader {
/**
@@ -28,7 +26,6 @@ public interface DictionaryCacheLoader {
*
* @param dictionaryInfo dictionary info object which will hold the required data
* for a given column
- * @param columnIdentifier column unique identifier
* @param dictionaryChunkStartOffset start offset from where dictionary file has to
* be read
* @param dictionaryChunkEndOffset end offset till where dictionary file has to
@@ -37,7 +34,7 @@ public interface DictionaryCacheLoader {
* read in memory after dictionary loading
* @throws IOException
*/
- void load(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier,
- long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
+ void load(DictionaryInfo dictionaryInfo, long dictionaryChunkStartOffset,
+ long dictionaryChunkEndOffset, boolean loadSortIndex)
throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
index a603c01..899abf5 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
@@ -23,7 +23,6 @@ import java.util.Iterator;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.reader.CarbonDictionaryReader;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
import org.apache.carbondata.core.service.CarbonCommonFactory;
@@ -43,8 +42,7 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
/**
* @param dictionaryColumnUniqueIdentifier dictionary column identifier
*/
- public DictionaryCacheLoaderImpl(
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+ DictionaryCacheLoaderImpl(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
this.dictionaryColumnUniqueIdentifier = dictionaryColumnUniqueIdentifier;
}
@@ -53,7 +51,6 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
*
* @param dictionaryInfo dictionary info object which will hold the required data
* for a given column
- * @param columnIdentifier column unique identifier
* @param dictionaryChunkStartOffset start offset from where dictionary file has to
* be read
* @param dictionaryChunkEndOffset end offset till where dictionary file has to
@@ -62,9 +59,9 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
* read in memory after dictionary loading
* @throws IOException
*/
- @Override public void load(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier,
- long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
- throws IOException {
+ @Override
+ public void load(DictionaryInfo dictionaryInfo, long dictionaryChunkStartOffset,
+ long dictionaryChunkEndOffset, boolean loadSortIndex) throws IOException {
Iterator<byte[]> columnDictionaryChunkWrapper =
load(dictionaryColumnUniqueIdentifier, dictionaryChunkStartOffset,
dictionaryChunkEndOffset);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java
index 3bfbc45..381e764 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java
@@ -43,16 +43,10 @@ public class BTreeBuilderInfo {
this.footerList = footerList;
}
- /**
- * @return the eachDimensionBlockSize
- */
public int[] getDimensionColumnValueSize() {
return dimensionColumnValueSize;
}
- /**
- * @return the footerList
- */
public List<DataFileFooter> getFooterList() {
return footerList;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 13d5f69..273f833 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -18,7 +18,6 @@ package org.apache.carbondata.core.datastore;
import java.io.IOException;
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
@@ -28,46 +27,47 @@ import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
public interface DataRefNode {
/**
- * Method to get the next block this can be used while scanning when
+ * Return the next data block in the tree, this can be used while scanning when
* iterator of this class can be used iterate over blocks
- *
- * @return next block
*/
DataRefNode getNextDataRefNode();
/**
- * to get the number of keys tuples present in the block
- *
- * @return number of keys in the block
+ * Return the number of rows in the data block
*/
- int nodeSize();
+ int numRows();
/**
- * Method can be used to get the block index .This can be used when multiple
- * thread can be used scan group of blocks in that can we can assign the
+ * Return the block index. This can be used when multiple
+ * thread can be used scan group of blocks in that can we can assign
* some of the blocks to one thread and some to other
- *
- * @return block number
*/
- long nodeNumber();
+ long nodeIndex();
+
+ /**
+ * Return the blocklet index in the node
+ */
+ short blockletIndex();
/**
- * Method is used for retreiving the BlockletId.
- * @return the blockletid related to the data block.
+ * Return the number of pages
*/
- String blockletId();
+ int numberOfPages();
/**
- * This method will be used to get the max value of all the columns this can
+ * Return the number of rows for a give page
+ */
+ int getPageRowCount(int pageNumber);
+
+ /**
+ * Return the max value of all the columns, this can
* be used in case of filter query
- *
*/
byte[][] getColumnsMaxValue();
/**
- * This method will be used to get the min value of all the columns this can
+ * Return the min value of all the columns, this can
* be used in case of filter query
- *
*/
byte[][] getColumnsMinValue();
@@ -75,15 +75,15 @@ public interface DataRefNode {
* Below method will be used to get the dimension chunks
*
* @param fileReader file reader to read the chunks from file
- * @param blockIndexes range indexes of the blocks need to be read
+ * @param columnIndexRange range indexes of the blocks need to be read
* value can be {{0,10},{11,12},{13,13}}
* here 0 to 10 and 11 to 12 column blocks will be read in one
* IO operation 13th column block will be read separately
* This will be helpful to reduce IO by reading bigger chunk of
- * data in On IO
+ * data in one IO operation
* @return dimension data chunks
*/
- DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader, int[][] blockIndexes)
+ DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader, int[][] columnIndexRange)
throws IOException;
/**
@@ -92,54 +92,31 @@ public interface DataRefNode {
* @param fileReader file reader to read the chunk from file
* @return dimension data chunk
*/
- DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndexes)
+ DimensionRawColumnChunk readDimensionChunk(FileReader fileReader, int columnIndex)
throws IOException;
/**
* Below method will be used to get the measure chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndexes range indexes of the blocks need to be read
+ * @param columnIndexRange range indexes of the blocks need to be read
* value can be {{0,10},{11,12},{13,13}}
* here 0 to 10 and 11 to 12 column blocks will be read in one
* IO operation 13th column block will be read separately
* This will be helpful to reduce IO by reading bigger chunk of
- * data in On IO
+ * data in one IO operation
* @return measure column data chunk
*/
- MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
+ MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader, int[][] columnIndexRange)
throws IOException;
/**
* Below method will be used to read the measure chunk
*
* @param fileReader file read to read the file chunk
- * @param blockIndex block index to be read from file
+ * @param columnIndex block index to be read from file
* @return measure data chunk
*/
- MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex) throws IOException;
+ MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex) throws IOException;
- /**
- * @param deleteDeltaDataCache
- */
- void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache);
-
- /**
- * @return
- */
- BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache();
-
- /**
- * number of pages in blocklet
- * @return
- */
- int numberOfPages();
-
- /**
- * Return the number of rows for a give page
- *
- * @param pageNumber
- * @return
- */
- int getPageRowCount(int pageNumber);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
deleted file mode 100644
index b4130a0..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public interface FileHolder {
-
- /**
- * This method will be used to reads the data to byteBuffer from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return ByteBuffer
- * @throws IOException
- */
- ByteBuffer readByteBuffer(String filePath, long offset, int length)
- throws IOException;
- /**
- * This method will be used to read the byte array from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, long offset, int length) throws IOException;
-
- /**
- * This method will be used to read the byte array from file based on length(number of bytes)
- *
- * @param filePath fully qualified file path
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, int length) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- int readInt(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readLong(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @return read int
- */
- int readInt(String filePath) throws IOException;
-
- /**
- * This method will be used to read long value from file from postion(offset), here
- * length will be always 8 because long byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readDouble(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to close all the streams currently present in the cache
- */
- void finish() throws IOException;
-
- void setQueryId(String queryId);
-
- String getQueryId();
-
- /**
- * Set the flag to read data page by page instead of whole blocklet.
- *
- * @param isReadPageByPage
- */
- void setReadPageByPage(boolean isReadPageByPage);
-
- /**
- * Whether read the data page by page from carbondata file instead of reading whole
- * blocklet to memory. It is need in case of memory constraint operations.
- */
- boolean isReadPageByPage();
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
new file mode 100644
index 0000000..df0d745
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface FileReader {
+
+ /**
+ * This method will be used to reads the data to byteBuffer from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return ByteBuffer
+ * @throws IOException
+ */
+ ByteBuffer readByteBuffer(String filePath, long offset, int length)
+ throws IOException;
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, long offset, int length) throws IOException;
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, int length) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ int readInt(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readLong(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ int readInt(String filePath) throws IOException;
+
+ /**
+ * This method will be used to read long value from file from postion(offset), here
+ * length will be always 8 because long byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readDouble(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ void finish() throws IOException;
+
+ /**
+ * Set the flag to read data page by page instead of whole blocklet.
+ *
+ * @param isReadPageByPage
+ */
+ void setReadPageByPage(boolean isReadPageByPage);
+
+ /**
+ * Whether read the data page by page from carbondata file instead of reading whole
+ * blocklet to memory. It is need in case of memory constraint operations.
+ */
+ boolean isReadPageByPage();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 5f82e87..c134db9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -94,10 +94,9 @@ public class SegmentProperties {
private int[] complexDimColumnCardinality;
/**
- * mapping of dimension column to block in a file this will be used for
- * reading the blocks from file
+ * mapping of dimension ordinal in schema to column chunk index in the data file
*/
- private Map<Integer, Integer> dimensionOrdinalToBlockMapping;
+ private Map<Integer, Integer> dimensionOrdinalToChunkMapping;
/**
* a block can have multiple columns. This will have block index as key
@@ -106,10 +105,9 @@ public class SegmentProperties {
private Map<Integer, Set<Integer>> blockTodimensionOrdinalMapping;
/**
- * mapping of measure column to block to in file this will be used while
- * reading the block in a file
+ * mapping of measure ordinal in schema to column chunk index in the data file
*/
- private Map<Integer, Integer> measuresOrdinalToBlockMapping;
+ private Map<Integer, Integer> measuresOrdinalToChunkMapping;
/**
* size of the each dimension column value in a block this can be used when
@@ -172,15 +170,15 @@ public class SegmentProperties {
new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
measures = new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
fillDimensionAndMeasureDetails(columnsInTable, columnCardinality);
- dimensionOrdinalToBlockMapping =
+ dimensionOrdinalToChunkMapping =
new HashMap<Integer, Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
blockTodimensionOrdinalMapping =
new HashMap<Integer, Set<Integer>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- measuresOrdinalToBlockMapping =
+ measuresOrdinalToChunkMapping =
new HashMap<Integer, Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
intialiseColGroups();
fillOrdinalToBlockMappingForDimension();
- fillOrdinalToBlockIndexMappingForMeasureColumns();
+ fillOrdinalToChunkIndexMappingForMeasureColumns();
fillColumnGroupAndItsCardinality(columnCardinality);
fillKeyGeneratorDetails();
}
@@ -237,7 +235,7 @@ public class SegmentProperties {
if (dimension.isColumnar() || dimension.columnGroupId() != prvcolumnGroupId) {
blockOrdinal++;
}
- dimensionOrdinalToBlockMapping.put(dimension.getOrdinal(), blockOrdinal);
+ dimensionOrdinalToChunkMapping.put(dimension.getOrdinal(), blockOrdinal);
prvcolumnGroupId = dimension.columnGroupId();
index++;
}
@@ -245,7 +243,7 @@ public class SegmentProperties {
// complex dimension will be stored at last
while (index < complexDimensions.size()) {
dimension = complexDimensions.get(index);
- dimensionOrdinalToBlockMapping.put(dimension.getOrdinal(), ++blockOrdinal);
+ dimensionOrdinalToChunkMapping.put(dimension.getOrdinal(), ++blockOrdinal);
blockOrdinal = fillComplexDimensionChildBlockIndex(blockOrdinal, dimension);
index++;
}
@@ -256,7 +254,7 @@ public class SegmentProperties {
*
*/
private void fillBlockToDimensionOrdinalMapping() {
- Set<Entry<Integer, Integer>> blocks = dimensionOrdinalToBlockMapping.entrySet();
+ Set<Entry<Integer, Integer>> blocks = dimensionOrdinalToChunkMapping.entrySet();
Iterator<Entry<Integer, Integer>> blockItr = blocks.iterator();
while (blockItr.hasNext()) {
Entry<Integer, Integer> block = blockItr.next();
@@ -280,7 +278,7 @@ public class SegmentProperties {
*/
private int fillComplexDimensionChildBlockIndex(int blockOrdinal, CarbonDimension dimension) {
for (int i = 0; i < dimension.getNumberOfChild(); i++) {
- dimensionOrdinalToBlockMapping
+ dimensionOrdinalToChunkMapping
.put(dimension.getListOfChildDimensions().get(i).getOrdinal(), ++blockOrdinal);
if (dimension.getListOfChildDimensions().get(i).getNumberOfChild() > 0) {
blockOrdinal = fillComplexDimensionChildBlockIndex(blockOrdinal,
@@ -295,11 +293,11 @@ public class SegmentProperties {
* of measure ordinal to its block index mapping in
* file
*/
- private void fillOrdinalToBlockIndexMappingForMeasureColumns() {
+ private void fillOrdinalToChunkIndexMappingForMeasureColumns() {
int blockOrdinal = 0;
int index = 0;
while (index < measures.size()) {
- measuresOrdinalToBlockMapping.put(measures.get(index).getOrdinal(), blockOrdinal);
+ measuresOrdinalToChunkMapping.put(measures.get(index).getOrdinal(), blockOrdinal);
blockOrdinal++;
index++;
}
@@ -731,17 +729,17 @@ public class SegmentProperties {
}
/**
- * @return the dimensionOrdinalToBlockMapping
+ * @return the dimensionOrdinalToChunkMapping
*/
- public Map<Integer, Integer> getDimensionOrdinalToBlockMapping() {
- return dimensionOrdinalToBlockMapping;
+ public Map<Integer, Integer> getDimensionOrdinalToChunkMapping() {
+ return dimensionOrdinalToChunkMapping;
}
/**
- * @return the measuresOrdinalToBlockMapping
+ * @return the measuresOrdinalToChunkMapping
*/
- public Map<Integer, Integer> getMeasuresOrdinalToBlockMapping() {
- return measuresOrdinalToBlockMapping;
+ public Map<Integer, Integer> getMeasuresOrdinalToChunkMapping() {
+ return measuresOrdinalToChunkMapping;
}
/**
@@ -805,16 +803,6 @@ public class SegmentProperties {
}
/**
- * It returns no of column availble in given column group
- *
- * @param colGrpId
- * @return no of column in given column group
- */
- public int getNoOfColumnsInColumnGroup(int colGrpId) {
- return columnGroupOrdinalToMdkeymapping.get(colGrpId).size();
- }
-
- /**
* @param blockIndex
* @return It returns all dimension present in given block index
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
deleted file mode 100644
index 3791314..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk;
-
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-
-/**
- * Interface for dimension column chunk.
- */
-public interface DimensionColumnDataChunk {
-
- /**
- * Below method will be used to fill the data based on offset and row id
- *
- * @param data data to filed
- * @param offset offset from which data need to be filed
- * @return how many bytes was copied
- */
- int fillChunkData(byte[] data, int offset, int columnIndex, KeyStructureInfo restructuringInfo);
-
- /**
- * It uses to convert column data to dictionary integer value
- *
- * @param rowId
- * @param columnIndex
- * @param row
- * @param restructuringInfo @return
- */
- int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
- KeyStructureInfo restructuringInfo);
-
- /**
- * Fill the data to vector
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo);
-
- /**
- * Fill the data to vector
- * @param rowMapping
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo);
-
- /**
- * Below method to get the data based in row id
- *
- * @return chunk
- */
- byte[] getChunkData(int columnIndex);
-
- /**
- * @return inverted index
- */
- int getInvertedIndex(int index);
-
- /**
- *
- * @param invertedIndex
- * @return index reverse index
- */
- int getInvertedReverseIndex(int invertedIndex);
-
- /**
- * @return whether column is dictionary column or not
- */
- boolean isNoDicitionaryColumn();
-
- /**
- * @return length of each column
- */
- int getColumnValueSize();
-
- /**
- * @return whether columns where explictly sorted or not
- */
- boolean isExplicitSorted();
-
- /**
- * to compare the data
- *
- * @param index row index to be compared
- * @param compareValue value to compare
- * @return compare result
- */
- int compareTo(int index, byte[] compareValue);
-
- /**
- * below method will be used to free the allocated memory
- */
- void freeMemory();
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
new file mode 100644
index 0000000..15840bc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk;
+
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * Interface for dimension column chunk.
+ */
+public interface DimensionColumnPage {
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param offset offset from which data need to be filed
+ * @param data data to filed
+ * @return how many bytes was copied
+ */
+ int fillRawData(int rowId, int offset, byte[] data, KeyStructureInfo restructuringInfo);
+
+ /**
+ * It uses to convert column data to dictionary integer value
+ *
+ * @param rowId
+ * @param chunkIndex
+ * @param outputSurrogateKey
+ * @param restructuringInfo @return
+ */
+ int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
+ KeyStructureInfo restructuringInfo);
+
+ /**
+ * Fill the data to vector
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo);
+
+ /**
+ * Fill the data to vector
+ * @param filteredRowId
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo);
+
+ /**
+ * Below method to get the data based in row id
+ *
+ * @return chunk
+ */
+ byte[] getChunkData(int rowId);
+
+ /**
+ * @return inverted index
+ */
+ int getInvertedIndex(int rowId);
+
+ /**
+ *
+ * @param rowId
+ * @return index reverse index
+ */
+ int getInvertedReverseIndex(int rowId);
+
+ /**
+ * @return whether column is dictionary column or not
+ */
+ boolean isNoDicitionaryColumn();
+
+ /**
+ * @return whether columns where explictly sorted or not
+ */
+ boolean isExplicitSorted();
+
+ /**
+ * to compare the data
+ *
+ * @param rowId row index to be compared
+ * @param compareValue value to compare
+ * @return compare result
+ */
+ int compareTo(int rowId, byte[] compareValue);
+
+ /**
+ * below method will be used to free the allocated memory
+ */
+ void freeMemory();
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionColumnPage.java
new file mode 100644
index 0000000..6f316c5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionColumnPage.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.impl;
+
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionDataChunkStore;
+
+/**
+ * Class responsibility is to give access to dimension column data chunk store
+ */
+public abstract class AbstractDimensionColumnPage implements DimensionColumnPage {
+
+ /**
+ * data chunks
+ */
+ DimensionDataChunkStore dataChunkStore;
+
+ /**
+ * @return whether columns where explicitly sorted or not
+ */
+ @Override public boolean isExplicitSorted() {
+ return dataChunkStore.isExplicitSorted();
+ }
+
+ /**
+ * Below method to get the data based in row id
+ *
+ * @param rowId row id of the data
+ * @return chunk
+ */
+ @Override public byte[] getChunkData(int rowId) {
+ return dataChunkStore.getRow(rowId);
+ }
+
+ /**
+ * @return inverted index
+ */
+ @Override public int getInvertedIndex(int rowId) {
+ return dataChunkStore.getInvertedIndex(rowId);
+ }
+
+ /**
+ * @param rowId
+ * @return inverted index reverse
+ */
+ @Override public int getInvertedReverseIndex(int rowId) {
+ return dataChunkStore.getInvertedReverseIndex(rowId);
+ }
+
+ /**
+ * To compare the data
+ *
+ * @param rowId row index to be compared
+ * @param compareValue value to compare
+ * @return compare result
+ */
+ @Override public int compareTo(int rowId, byte[] compareValue) {
+ // TODO Auto-generated method stub
+ return dataChunkStore.compareTo(rowId, compareValue);
+ }
+
+ /**
+ * below method will be used to free the allocated memory
+ */
+ @Override public void freeMemory() {
+ dataChunkStore.freeMemory();
+ }
+
+ /**
+ * @return column is dictionary column or not
+ */
+ @Override public boolean isNoDicitionaryColumn() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
deleted file mode 100644
index eac062f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionDataChunkStore;
-
-/**
- * Class responsibility is to give access to dimension column data chunk store
- */
-public abstract class AbstractDimensionDataChunk implements DimensionColumnDataChunk {
-
- /**
- * data chunks
- */
- protected DimensionDataChunkStore dataChunkStore;
-
- /**
- * @return whether columns where explicitly sorted or not
- */
- @Override public boolean isExplicitSorted() {
- return dataChunkStore.isExplicitSorted();
- }
-
- /**
- * Below method to get the data based in row id
- *
- * @param index row id of the data
- * @return chunk
- */
- @Override public byte[] getChunkData(int index) {
- return dataChunkStore.getRow(index);
- }
-
- /**
- * @return inverted index
- */
- @Override public int getInvertedIndex(int index) {
- return dataChunkStore.getInvertedIndex(index);
- }
-
- /**
- * @param invertedIndex
- * @return inverted index reverse
- */
- @Override public int getInvertedReverseIndex(int invertedIndex) {
- return dataChunkStore.getInvertedReverseIndex(invertedIndex);
- }
- /**
- * @return length of each column
- */
- @Override public int getColumnValueSize() {
- return dataChunkStore.getColumnValueSize();
- }
-
- /**
- * To compare the data
- *
- * @param index row index to be compared
- * @param compareValue value to compare
- * @return compare result
- */
- @Override public int compareTo(int index, byte[] compareValue) {
- // TODO Auto-generated method stub
- return dataChunkStore.compareTo(index, compareValue);
- }
-
- /**
- * below method will be used to free the allocated memory
- */
- @Override public void freeMemory() {
- dataChunkStore.freeMemory();
- }
-
- /**
- * @return column is dictionary column or not
- */
- @Override public boolean isNoDicitionaryColumn() {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionColumnPage.java
new file mode 100644
index 0000000..741c13d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionColumnPage.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.impl;
+
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * This class is gives access to column group dimension data chunk store
+ */
+public class ColumnGroupDimensionColumnPage extends AbstractDimensionColumnPage {
+
+ /**
+ * Constructor for this class
+ *
+ * @param dataChunk data chunk
+ * @param columnValueSize chunk attributes
+ * @param numberOfRows
+ */
+ public ColumnGroupDimensionColumnPage(byte[] dataChunk, int columnValueSize, int numberOfRows) {
+ this.dataChunkStore = DimensionChunkStoreFactory.INSTANCE
+ .getDimensionChunkStore(columnValueSize, false, numberOfRows, dataChunk.length,
+ DimensionStoreType.FIXEDLENGTH);
+ this.dataChunkStore.putArray(null, null, dataChunk);
+ }
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param rowId row id of the chunk
+ * @param offset offset from which data need to be filed
+ * @param data data to filed
+ * @param restructuringInfo define the structure of the key
+ * @return how many bytes was copied
+ */
+ @Override public int fillRawData(int rowId, int offset, byte[] data,
+ KeyStructureInfo restructuringInfo) {
+ byte[] row = dataChunkStore.getRow(rowId);
+ byte[] maskedKey = getMaskedKey(row, restructuringInfo);
+ System.arraycopy(maskedKey, 0, data, offset, maskedKey.length);
+ return maskedKey.length;
+ }
+
+ /**
+ * Converts to column dictionary integer value
+ *
+ * @param rowId
+ * @param chunkIndex
+ * @param outputSurrogateKey
+ * @param info KeyStructureInfo
+ * @return
+ */
+ @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
+ KeyStructureInfo info) {
+ byte[] data = dataChunkStore.getRow(rowId);
+ long[] keyArray = info.getKeyGenerator().getKeyArray(data);
+ int[] ordinal = info.getMdkeyQueryDimensionOrdinal();
+ for (int i = 0; i < ordinal.length; i++) {
+ outputSurrogateKey[chunkIndex++] = (int) keyArray[ordinal[i]];
+ }
+ return chunkIndex;
+ }
+
+ /**
+ * Below method will be used to get the masked key
+ *
+ * @param data data
+ * @param info
+ * @return
+ */
+ private byte[] getMaskedKey(byte[] data, KeyStructureInfo info) {
+ byte[] maskedKey = new byte[info.getMaskByteRanges().length];
+ int counter = 0;
+ int byteRange = 0;
+ for (int i = 0; i < info.getMaskByteRanges().length; i++) {
+ byteRange = info.getMaskByteRanges()[i];
+ maskedKey[counter++] = (byte) (data[byteRange] & info.getMaxKey()[byteRange]);
+ }
+ return maskedKey;
+ }
+
+ /**
+ * @return inverted index
+ */
+ @Override public int getInvertedIndex(int rowId) {
+ throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
+ }
+
+ /**
+ * @param rowId
+ * @return inverted index reverse
+ */
+ @Override public int getInvertedReverseIndex(int rowId) {
+ throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
+ }
+
+ /**
+ * @return whether columns where explictly sorted or not
+ */
+ @Override public boolean isExplicitSorted() {
+ return false;
+ }
+
+ /**
+ * to compare the data
+ *
+ * @param rowId row index to be compared
+ * @param compareValue value to compare
+ * @return compare result
+ */
+ @Override public int compareTo(int rowId, byte[] compareValue) {
+ throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ int[] ordinal = restructuringInfo.getMdkeyQueryDimensionOrdinal();
+ for (int k = offset; k < len; k++) {
+ long[] keyArray = restructuringInfo.getKeyGenerator().getKeyArray(dataChunkStore.getRow(k));
+ int index = 0;
+ for (int i = chunkIndex; i < chunkIndex + ordinal.length; i++) {
+ if (vectorInfo[i].directDictionaryGenerator == null) {
+ vectorInfo[i].vector.putInt(vectorOffset, (int) keyArray[ordinal[index++]]);
+ } else {
+ vectorInfo[i].vector.putLong(vectorOffset, (long) vectorInfo[i].directDictionaryGenerator
+ .getValueFromSurrogate((int) keyArray[ordinal[index++]]));
+ }
+ }
+ vectorOffset++;
+ }
+ return chunkIndex + ordinal.length;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param filteredRowId
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo,
+ int chunkIndex, KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ int[] ordinal = restructuringInfo.getMdkeyQueryDimensionOrdinal();
+ for (int k = offset; k < len; k++) {
+ long[] keyArray =
+ restructuringInfo.getKeyGenerator().getKeyArray(dataChunkStore.getRow(filteredRowId[k]));
+ int index = 0;
+ for (int i = chunkIndex; i < chunkIndex + ordinal.length; i++) {
+ if (vectorInfo[i].directDictionaryGenerator == null) {
+ vectorInfo[i].vector.putInt(vectorOffset, (int) keyArray[ordinal[index++]]);
+ } else {
+ vectorInfo[i].vector.putLong(vectorOffset, (long) vectorInfo[i].directDictionaryGenerator
+ .getValueFromSurrogate((int) keyArray[ordinal[index++]]));
+ }
+ }
+ vectorOffset++;
+ }
+ return chunkIndex + ordinal.length;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
deleted file mode 100644
index b76ae53..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-
-/**
- * This class is gives access to column group dimension data chunk store
- */
-public class ColumnGroupDimensionDataChunk extends AbstractDimensionDataChunk {
-
- /**
- * Constructor for this class
- *
- * @param dataChunk data chunk
- * @param columnValueSize chunk attributes
- * @param numberOfRows
- */
- public ColumnGroupDimensionDataChunk(byte[] dataChunk, int columnValueSize, int numberOfRows) {
- this.dataChunkStore = DimensionChunkStoreFactory.INSTANCE
- .getDimensionChunkStore(columnValueSize, false, numberOfRows, dataChunk.length,
- DimensionStoreType.FIXEDLENGTH);
- this.dataChunkStore.putArray(null, null, dataChunk);
- }
-
- /**
- * Below method will be used to fill the data based on offset and row id
- *
- * @param data data to filed
- * @param offset offset from which data need to be filed
- * @param rowId row id of the chunk
- * @param restructuringInfo define the structure of the key
- * @return how many bytes was copied
- */
- @Override public int fillChunkData(byte[] data, int offset, int rowId,
- KeyStructureInfo restructuringInfo) {
- byte[] row = dataChunkStore.getRow(rowId);
- byte[] maskedKey = getMaskedKey(row, restructuringInfo);
- System.arraycopy(maskedKey, 0, data, offset, maskedKey.length);
- return maskedKey.length;
- }
-
- /**
- * Converts to column dictionary integer value
- *
- * @param rowId
- * @param columnIndex
- * @param row
- * @param info KeyStructureInfo
- * @return
- */
- @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
- KeyStructureInfo info) {
- byte[] data = dataChunkStore.getRow(rowId);
- long[] keyArray = info.getKeyGenerator().getKeyArray(data);
- int[] ordinal = info.getMdkeyQueryDimensionOrdinal();
- for (int i = 0; i < ordinal.length; i++) {
- row[columnIndex++] = (int) keyArray[ordinal[i]];
- }
- return columnIndex;
- }
-
- /**
- * Below method will be used to get the masked key
- *
- * @param data data
- * @param info
- * @return
- */
- private byte[] getMaskedKey(byte[] data, KeyStructureInfo info) {
- byte[] maskedKey = new byte[info.getMaskByteRanges().length];
- int counter = 0;
- int byteRange = 0;
- for (int i = 0; i < info.getMaskByteRanges().length; i++) {
- byteRange = info.getMaskByteRanges()[i];
- maskedKey[counter++] = (byte) (data[byteRange] & info.getMaxKey()[byteRange]);
- }
- return maskedKey;
- }
-
- /**
- * @return inverted index
- */
- @Override public int getInvertedIndex(int index) {
- throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
- }
-
- /**
- * @param invertedIndex
- * @return inverted index reverse
- */
- @Override public int getInvertedReverseIndex(int invertedIndex) {
- throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
- }
-
- /**
- * @return whether columns where explictly sorted or not
- */
- @Override public boolean isExplicitSorted() {
- return false;
- }
-
- /**
- * to compare the data
- *
- * @param index row index to be compared
- * @param compareValue value to compare
- * @return compare result
- */
- @Override public int compareTo(int index, byte[] compareValue) {
- throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
- }
-
- /**
- * Fill the data to vector
- *
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- int[] ordinal = restructuringInfo.getMdkeyQueryDimensionOrdinal();
- for (int k = offset; k < len; k++) {
- long[] keyArray = restructuringInfo.getKeyGenerator().getKeyArray(dataChunkStore.getRow(k));
- int index = 0;
- for (int i = column; i < column + ordinal.length; i++) {
- if (vectorInfo[i].directDictionaryGenerator == null) {
- vectorInfo[i].vector.putInt(vectorOffset, (int) keyArray[ordinal[index++]]);
- } else {
- vectorInfo[i].vector.putLong(vectorOffset, (long) vectorInfo[i].directDictionaryGenerator
- .getValueFromSurrogate((int) keyArray[ordinal[index++]]));
- }
- }
- vectorOffset++;
- }
- return column + ordinal.length;
- }
-
- /**
- * Fill the data to vector
- *
- * @param rowMapping
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo,
- int column, KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- int[] ordinal = restructuringInfo.getMdkeyQueryDimensionOrdinal();
- for (int k = offset; k < len; k++) {
- long[] keyArray =
- restructuringInfo.getKeyGenerator().getKeyArray(dataChunkStore.getRow(rowMapping[k]));
- int index = 0;
- for (int i = column; i < column + ordinal.length; i++) {
- if (vectorInfo[i].directDictionaryGenerator == null) {
- vectorInfo[i].vector.putInt(vectorOffset, (int) keyArray[ordinal[index++]]);
- } else {
- vectorInfo[i].vector.putLong(vectorOffset, (long) vectorInfo[i].directDictionaryGenerator
- .getValueFromSurrogate((int) keyArray[ordinal[index++]]));
- }
- }
- vectorOffset++;
- }
- return column + ordinal.length;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
index 705c13c..f9bb590 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
@@ -19,25 +19,25 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.memory.MemoryException;
/**
* Contains raw dimension data,
* 1. The read uncompressed raw data of column chunk with all pages is stored in this instance.
- * 2. The raw data can be converted to processed chunk using convertToDimColDataChunk method
+ * 2. The raw data can be converted to processed chunk using decodeColumnPage method
* by specifying page number.
*/
public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
- private DimensionColumnDataChunk[] dataChunks;
+ private DimensionColumnPage[] dataChunks;
private DimensionColumnChunkReader chunkReader;
- private FileHolder fileHolder;
+ private FileReader fileReader;
public DimensionRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
DimensionColumnChunkReader columnChunkReader) {
@@ -46,17 +46,17 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
}
/**
- * Convert all raw data with all pages to processed DimensionColumnDataChunk's
+ * Convert all raw data with all pages to processed DimensionColumnPage's
* @return
*/
- public DimensionColumnDataChunk[] convertToDimColDataChunks() {
+ public DimensionColumnPage[] decodeAllColumnPages() {
if (dataChunks == null) {
- dataChunks = new DimensionColumnDataChunk[pagesCount];
+ dataChunks = new DimensionColumnPage[pagesCount];
}
for (int i = 0; i < pagesCount; i++) {
try {
if (dataChunks[i] == null) {
- dataChunks[i] = chunkReader.convertToDimensionChunk(this, i);
+ dataChunks[i] = chunkReader.decodeColumnPage(this, i);
}
} catch (IOException | MemoryException e) {
throw new RuntimeException(e);
@@ -66,24 +66,24 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
}
/**
- * Convert raw data with specified page number processed to DimensionColumnDataChunk
- * @param index
+ * Convert raw data with specified page number processed to DimensionColumnPage
+ * @param pageNumber
* @return
*/
- public DimensionColumnDataChunk convertToDimColDataChunk(int index) {
- assert index < pagesCount;
+ public DimensionColumnPage decodeColumnPage(int pageNumber) {
+ assert pageNumber < pagesCount;
if (dataChunks == null) {
- dataChunks = new DimensionColumnDataChunk[pagesCount];
+ dataChunks = new DimensionColumnPage[pagesCount];
}
- if (dataChunks[index] == null) {
+ if (dataChunks[pageNumber] == null) {
try {
- dataChunks[index] = chunkReader.convertToDimensionChunk(this, index);
+ dataChunks[pageNumber] = chunkReader.decodeColumnPage(this, pageNumber);
} catch (IOException | MemoryException e) {
throw new RuntimeException(e);
}
}
- return dataChunks[index];
+ return dataChunks[pageNumber];
}
/**
@@ -92,10 +92,10 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
* @param index
* @return
*/
- public DimensionColumnDataChunk convertToDimColDataChunkWithOutCache(int index) {
+ public DimensionColumnPage convertToDimColDataChunkWithOutCache(int index) {
assert index < pagesCount;
try {
- return chunkReader.convertToDimensionChunk(this, index);
+ return chunkReader.decodeColumnPage(this, index);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -111,11 +111,11 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
}
}
- public void setFileHolder(FileHolder fileHolder) {
- this.fileHolder = fileHolder;
+ public void setFileReader(FileReader fileReader) {
+ this.fileReader = fileReader;
}
- public FileHolder getFileReader() {
- return fileHolder;
+ public FileReader getFileReader() {
+ return fileReader;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
new file mode 100644
index 0000000..ff54b12
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.impl;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * This class is gives access to fixed length dimension data chunk store
+ */
+public class FixedLengthDimensionColumnPage extends AbstractDimensionColumnPage {
+
+ /**
+ * Constructor
+ *
+ * @param dataChunk data chunk
+ * @param invertedIndex inverted index
+ * @param invertedIndexReverse reverse inverted index
+ * @param numberOfRows number of rows
+ * @param columnValueSize size of each column value
+ */
+ public FixedLengthDimensionColumnPage(byte[] dataChunk, int[] invertedIndex,
+ int[] invertedIndexReverse, int numberOfRows, int columnValueSize) {
+ long totalSize = null != invertedIndex ?
+ dataChunk.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) :
+ dataChunk.length;
+ dataChunkStore = DimensionChunkStoreFactory.INSTANCE
+ .getDimensionChunkStore(columnValueSize, null != invertedIndex, numberOfRows, totalSize,
+ DimensionStoreType.FIXEDLENGTH);
+ dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunk);
+ }
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param rowId row id of the chunk
+ * @param offset offset from which data need to be filed
+ * @param data data to filed
+ * @param keyStructureInfo define the structure of the key
+ * @return how many bytes was copied
+ */
+ @Override public int fillRawData(int rowId, int offset, byte[] data,
+ KeyStructureInfo keyStructureInfo) {
+ dataChunkStore.fillRow(rowId, data, offset);
+ return dataChunkStore.getColumnValueSize();
+ }
+
+ /**
+ * Converts to column dictionary integer value
+ *
+ * @param rowId
+ * @param chunkIndex
+ * @param outputSurrogateKey
+ * @param restructuringInfo
+ * @return
+ */
+ @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
+ KeyStructureInfo restructuringInfo) {
+ outputSurrogateKey[chunkIndex] = dataChunkStore.getSurrogate(rowId);
+ return chunkIndex + 1;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = columnVectorInfo.size + offset;
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ for (int j = offset; j < len; j++) {
+ int dict = dataChunkStore.getSurrogate(j);
+ if (columnVectorInfo.directDictionaryGenerator == null) {
+ vector.putInt(vectorOffset++, dict);
+ } else {
+ Object valueFromSurrogate =
+ columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict);
+ if (valueFromSurrogate == null) {
+ vector.putNull(vectorOffset++);
+ } else {
+ DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType();
+ if (dataType == DataTypes.INT) {
+ vector.putInt(vectorOffset++, (int) valueFromSurrogate);
+ } else if (dataType == DataTypes.LONG) {
+ vector.putLong(vectorOffset++, (long) valueFromSurrogate);
+ } else {
+ throw new IllegalArgumentException("unsupported data type: " +
+ columnVectorInfo.directDictionaryGenerator.getReturnType());
+ }
+ }
+ }
+ }
+ return chunkIndex + 1;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param filteredRowId
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo,
+ int chunkIndex, KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = columnVectorInfo.size + offset;
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ for (int j = offset; j < len; j++) {
+ int dict = dataChunkStore.getSurrogate(filteredRowId[j]);
+ if (columnVectorInfo.directDictionaryGenerator == null) {
+ vector.putInt(vectorOffset++, dict);
+ } else {
+ Object valueFromSurrogate =
+ columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict);
+ if (valueFromSurrogate == null) {
+ vector.putNull(vectorOffset++);
+ } else {
+ DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType();
+ if (dataType == DataTypes.INT) {
+ vector.putInt(vectorOffset++, (int) valueFromSurrogate);
+ } else if (dataType == DataTypes.LONG) {
+ vector.putLong(vectorOffset++, (long) valueFromSurrogate);
+ } else {
+ throw new IllegalArgumentException("unsupported data type: " +
+ columnVectorInfo.directDictionaryGenerator.getReturnType());
+ }
+ }
+ }
+ }
+ return chunkIndex + 1;
+ }
+}
[10/50] [abbrv] carbondata git commit: [CARBONDATA-2090] Fix the
error message of alter streaming property
Posted by ra...@apache.org.
[CARBONDATA-2090] Fix the error message of alter streaming property
Fix the error message of alter streaming property
This closes #1873
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c9a501dc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c9a501dc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c9a501dc
Branch: refs/heads/fgdatamap
Commit: c9a501dca672678093fe0e5f526b226057b96d5d
Parents: 4d3f398
Author: QiangCai <qi...@qq.com>
Authored: Mon Jan 29 11:40:47 2018 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jan 31 11:58:44 2018 +0800
----------------------------------------------------------------------
.../spark/sql/execution/strategy/DDLStrategy.scala | 7 ++++++-
.../spark/carbondata/TestStreamingTableOperation.scala | 11 ++++++++++-
2 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9a501dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 57be754..db8c6a2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -231,9 +231,14 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
// TODO remove this limitation later
val property = properties.find(_._1.equalsIgnoreCase("streaming"))
if (property.isDefined) {
- if (!property.get._2.trim.equalsIgnoreCase("true")) {
+ if (carbonTable.isStreamingTable) {
throw new MalformedCarbonCommandException(
"Streaming property can not be changed once it is 'true'")
+ } else {
+ if (!property.get._2.trim.equalsIgnoreCase("true")) {
+ throw new MalformedCarbonCommandException(
+ "Streaming property value is incorrect")
+ }
}
}
ExecutedCommandExec(CarbonAlterTableSetCommand(tableName, properties, isView)) :: Nil
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9a501dc/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 18e52f6..44204d4 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -181,7 +181,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
}
// normal table not support streaming ingest
- test("normal table not support streaming ingest") {
+ test("normal table not support streaming ingest and alter normal table's streaming property") {
+ // alter normal table's streaming property
+ val msg = intercept[MalformedCarbonCommandException](sql("alter table streaming.batch_table set tblproperties('streaming'='false')"))
+ assertResult("Streaming property value is incorrect")(msg.getMessage)
+
val identifier = new TableIdentifier("batch_table", Option("streaming"))
val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
@@ -518,6 +522,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
case _ =>
assert(false, "should support set table to streaming")
}
+
+ // alter streaming table's streaming property
+ val msg = intercept[MalformedCarbonCommandException](sql("alter table streaming.stream_table_handoff set tblproperties('streaming'='false')"))
+ assertResult("Streaming property can not be changed once it is 'true'")(msg.getMessage)
+
val segments = sql("show segments for table streaming.stream_table_handoff").collect()
assert(segments.length == 2 || segments.length == 3)
assertResult("Streaming")(segments(0).getString(1))
[23/50] [abbrv] carbondata git commit: [CARBONDATA-2089]SQL exception
is masked due to assert(false) inside try catch and exception block always
asserting true
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
index d6fa3ca..b4459ab 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
@@ -20,6 +20,8 @@ package org.apache.carbondata.cluster.sdv.generated
import java.sql.Timestamp
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util._
import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, BeforeAndAfterEach}
@@ -60,6 +62,9 @@ class DataLoadingIUDTestCase extends QueryTest with BeforeAndAfterAll with Befor
sql("drop table if exists t_carbn01b").collect
sql("drop table if exists T_Hive1").collect
sql("drop table if exists T_Hive6").collect
+ sql(s"""create table default.t_carbn01b(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' INTO table default.t_carbn01B options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='Active_status,Item_type_cd,Qty_day_avg,Qty_total,Sell_price,Sell_pricep,Discount_price,Profit,Item_code,Item_name,Outlet_name,Update_time,Create_date')""").collect
+
}
override def before(fun: => Any) {
@@ -75,9 +80,7 @@ class DataLoadingIUDTestCase extends QueryTest with BeforeAndAfterAll with Befor
//NA
test("IUD-01-01-01_001-001", Include) {
- sql(s"""create table default.t_carbn01b(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' INTO table default.t_carbn01B options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='Active_status,Item_type_cd,Qty_day_avg,Qty_total,Sell_price,Sell_pricep,Discount_price,Profit,Item_code,Item_name,Outlet_name,Update_time,Create_date')""").collect
- sql("create table T_Hive1(Active_status BOOLEAN, Item_type_cd TINYINT, Qty_day_avg SMALLINT, Qty_total INT, Sell_price BIGINT, Sell_pricep FLOAT, Discount_price DOUBLE , Profit DECIMAL(3,2), Item_code STRING, Item_name VARCHAR(50), Outlet_name CHAR(100), Update_time TIMESTAMP, Create_date DATE) row format delimited fields terminated by ',' collection items terminated by '$'")
+ sql("create table T_Hive1(Active_status BOOLEAN, Item_type_cd TINYINT, Qty_day_avg SMALLINT, Qty_total INT, Sell_price BIGINT, Sell_pricep FLOAT, Discount_price DOUBLE , Profit DECIMAL(3,2), Item_code STRING, Item_name VARCHAR(50), Outlet_name CHAR(100), Update_time TIMESTAMP, Create_date DATE) row format delimited fields terminated by ',' collection items terminated by '$'")
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/InsertData/T_Hive1.csv' overwrite into table T_Hive1""").collect
sql("create table T_Hive6(Item_code STRING, Sub_item_cd ARRAY<string>)row format delimited fields terminated by ',' collection items terminated by '$'")
sql(s"""load data inpath '$resourcesPath/Data/InsertData/T_Hive1.csv' overwrite into table T_Hive6""").collect
@@ -115,16 +118,13 @@ test("IUD-01-01-01_001-02", Include) {
//Check for update Carbon table using a data value on a string column without giving values in semi quote
test("IUD-01-01-01_001-03", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table IF EXISTS default.t_carbn01""").collect
sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
sql(s"""update default.t_carbn01 set (active_status) = (NO) """).collect
sql(s"""NA""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table default.t_carbn01 """).collect
}
@@ -204,18 +204,14 @@ test("IUD-01-01-01_001-11", Include) {
//Check for update Carbon table for a column where column name is mentioned incorrectly
test("IUD-01-01-01_001-14", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set (item_status_cd) = ('10')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set (item_status_cd) = ('10')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -245,35 +241,27 @@ test("IUD-01-01-01_001-16", Include) {
//Check for update Carbon table for a numeric value column using string value
test("IUD-01-01-01_001-17", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set (item_type_cd) = ('Orange')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set (item_type_cd) = ('Orange')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
//Check for update Carbon table for a numeric value column using decimal value
test("IUD-01-01-01_001-18", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set (item_type_cd) = ('10.11')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set (item_type_cd) = ('10.11')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -303,18 +291,14 @@ test("IUD-01-01-01_001-20", Include) {
//Check for update Carbon table for a numeric Int value column using large numeric value which is beyond 32 bit
test("IUD-01-01-01_001-21", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set (item_type_cd) = (-2147483649)""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set (item_type_cd) = (-2147483649)""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -380,18 +364,14 @@ test("IUD-01-01-01_001-26", Include) {
//Check for update Carbon table for a decimal value column using String value
test("IUD-01-01-01_001-27", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set (profit) = ('hakshk')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set (profit) = ('hakshk')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -445,86 +425,66 @@ test("IUD-01-01-01_001-31", Include) {
//Check for update Carbon table for a time stamp value column using date timestamp all formats.
test("IUD-01-01-01_001-35", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set(update_time) = ('04-11-20004 18:13:59.113')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set(update_time) = ('04-11-20004 18:13:59.113')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
//Check for update Carbon table for a time stamp value column using string value
test("IUD-01-01-01_001-32", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set(update_time) = ('fhjfhjfdshf')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set(update_time) = ('fhjfhjfdshf')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
//Check for update Carbon table for a time stamp value column using numeric
test("IUD-01-01-01_001-33", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set(update_time) = (56546)""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set(update_time) = (56546)""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
//Check for update Carbon table for a time stamp value column using date
test("IUD-01-01-01_001-34", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set(update_time) = ('2016-11-04')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set(update_time) = ('2016-11-04')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
//Check for update Carbon table for a time stamp value column using date timestamp
test("IUD-01-01-01_001-36", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set(update_time) = ('2016-11-04 18:63:59.113')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set(update_time) = ('2016-11-04 18:63:59.113')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -554,18 +514,14 @@ test("IUD-01-01-01_001-40", Include) {
//Check update Carbon table using a / operation on a column value
test("IUD-01-01-01_001-41", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set(item_type_cd)= (item_type_cd/1)""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set(item_type_cd)= (item_type_cd/1)""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -821,18 +777,14 @@ test("IUD-01-01-01_004-05", Include) {
//Check for update Carbon table where source table is having big int and target is having int value column for update
test("IUD-01-01-01_004-06", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 a set (a.item_type_cd) = (select b.sell_price from default.t_carbn01b b where b.sell_price=200000343430000000)""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 a set (a.item_type_cd) = (select b.sell_price from default.t_carbn01b b where b.sell_price=200000343430000000)""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -850,35 +802,27 @@ test("IUD-01-01-01_004-07", Include) {
//Check for update Carbon table where source table is having string and target is having decimal value column for update
test("IUD-01-01-01_004-08", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 a set (a.profit) = (select b.item_code from default.t_carbn01b b where b.item_code='DE3423ee')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 a set (a.profit) = (select b.item_code from default.t_carbn01b b where b.item_code='DE3423ee')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
//Check for update Carbon table where source table is having string and target is having timestamp column for update
test("IUD-01-01-01_004-09", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 a set (a.update_time) = (select b.item_code from default.t_carbn01b b where b.item_code='DE3423ee')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 a set (a.update_time) = (select b.item_code from default.t_carbn01b b where b.item_code='DE3423ee')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -968,17 +912,21 @@ test("IUD-01-01-01_005-12", Include) {
//Check for update Carbon table where a update column is dimension and is defined with exclude dictionary.
test("IUD-01-01-01_005-13", Include) {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Item_type_cd INT, Profit DECIMAL(3,2))STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='Item_type_cd')""").collect
- sql(s"""insert into default.t_carbn01 select item_type_cd, profit from default.t_carbn01b""").collect
-
- try {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Item_type_cd INT, Profit DECIMAL(3,2))STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='Item_type_cd')""").collect
+ sql(s"""insert into default.t_carbn01 select item_type_cd, profit from default.t_carbn01b""").collect
+ val currProperty = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
+ .CARBON_BAD_RECORDS_ACTION);
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FAIL")
+ intercept[Exception] {
sql(s"""update default.t_carbn01 set (item_type_cd) = ('ASASDDD')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, currProperty)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, currProperty)
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -1061,18 +1009,14 @@ test("IUD-01-01-01_009-01", Include) {
//Check update on carbon table using incorrect data value
test("IUD-01-01-01_010-01", Include) {
- try {
- sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
- sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""update default.t_carbn01 set Update_time = '11-11-2012 77:77:77') where item_code='ASD423ee')""").collect
+ intercept[Exception] {
+ sql(s"""drop table IF EXISTS default.t_carbn01 """).collect
+ sql(s"""create table default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""update default.t_carbn01 set Update_time = '11-11-2012 77:77:77') where item_code='ASD423ee')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -1586,17 +1530,13 @@ test("IUD-01-01-02_009-01", Include) {
//Check update on carbon table where a column being updated with incorrect data type.
test("IUD-01-01-02_011-01", Include) {
- try {
- sql(s"""create table if not exists default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""Update T_Carbn04 set (Item_type_cd) = ('Banana')""").collect
+ intercept[Exception] {
+ sql(s"""create table if not exists default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""Update T_Carbn04 set (Item_type_cd) = ('Banana')""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -1613,17 +1553,13 @@ test("IUD-01-01-01_022-01", Include) {
//Check update on carbon table where multiple values are returned in expression.
test("IUD-01-01-01_023-00", Include) {
- try {
- sql(s"""create table if not exists default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""Update default.t_carbn01 set Item_type_cd = (select Item_type_cd from default.t_carbn01b )""").collect
+ intercept[Exception] {
+ sql(s"""create table if not exists default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""Update default.t_carbn01 set Item_type_cd = (select Item_type_cd from default.t_carbn01b )""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
@@ -1643,17 +1579,13 @@ test("IUD-01-01-02_023-01", Include) {
//Check update on carbon table where non matching values are returned from expression.
test("IUD-01-01-01_024-01", Include) {
- try {
- sql(s"""create table if not exists default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
- sql(s"""Update default.t_carbn01 set Item_type_cd = (select Item_code from default.t_carbn01b)""").collect
+ intercept[Exception] {
+ sql(s"""create table if not exists default.t_carbn01 (Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""").collect
+ sql(s"""insert into default.t_carbn01 select * from default.t_carbn01b""").collect
+ sql(s"""Update default.t_carbn01 set Item_type_cd = (select Item_code from default.t_carbn01b)""").collect
sql(s"""NA""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table default.t_carbn01 """).collect
+ sql(s"""drop table default.t_carbn01 """).collect
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
index 8ff47af..52396ee 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
@@ -124,7 +124,7 @@ class DataLoadingTestCase extends QueryTest with BeforeAndAfterAll {
//Data load-->Empty BadRecords Parameters
test("BadRecord_Dataload_011", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE badrecords_test1 (ID int,CUST_ID int,sal int,cust_name string) STORED BY 'org.apache.carbondata.format'""")
.collect
@@ -133,11 +133,8 @@ class DataLoadingTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(
s"""select count(*) from badrecords_test1""",
Seq(Row(0)), "DataLoadingTestCase-BadRecord_Dataload_011")
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table badrecords_test1""").collect
+ sql(s"""drop table badrecords_test1""").collect
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/InvertedindexTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/InvertedindexTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/InvertedindexTestCase.scala
index bae0124..d9d35fb 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/InvertedindexTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/InvertedindexTestCase.scala
@@ -886,17 +886,13 @@ class InvertedindexTestCase extends QueryTest with BeforeAndAfterAll {
//to check alter drop column for no_inverted
test("NoInvertedindex-TC097", Include) {
sql(s"""drop table if exists uniqdata""").collect
- try {
- sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('COLUMN_GROUPS'='(CUST_NAME,ACTIVE_EMUI_VERSION)','DICTIONARY_INCLUDE'='CUST_ID','NO_INVERTED_INDEX'='CUST_NAME')""").collect
- sql(s"""Alter table uniqdata drop columns(BIGINT_COLUMN1)""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/noinverted.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ intercept[Exception] {
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('COLUMN_GROUPS'='(CUST_NAME,ACTIVE_EMUI_VERSION)','DICTIONARY_INCLUDE'='CUST_ID','NO_INVERTED_INDEX'='CUST_NAME')""").collect
+ sql(s"""Alter table uniqdata drop columns(BIGINT_COLUMN1)""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/noinverted.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
sql(s"""select BIGINT_COLUMN1 from uniqdata""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists uniqdata""").collect
+ sql(s"""drop table if exists uniqdata""").collect
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery1TestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery1TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery1TestCase.scala
index d93b2ee..e213e49 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery1TestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery1TestCase.scala
@@ -44,15 +44,9 @@ test("OffHeapQuery-001-TC_001", Include) {
//To check select query with limit as string
test("OffHeapQuery-001-TC_002", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 limit """"").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -112,57 +106,33 @@ test("OffHeapQuery-001-TC_008", Include) {
//To check where clause with OR and no operand
test("OffHeapQuery-001-TC_009", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id > 1 OR """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check OR clause with LHS and RHS having no arguments
test("OffHeapQuery-001-TC_010", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where OR """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check OR clause with LHS having no arguments
test("OffHeapQuery-001-TC_011", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where OR cust_id > "1"""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check incorrect query
test("OffHeapQuery-001-TC_013", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id > 0 OR name """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -231,15 +201,9 @@ test("OffHeapQuery-001-TC_020", Include) {
//To check select count and distinct query execution
test("OffHeapQuery-001-TC_021", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select count(cust_id),distinct(cust_name) from uniqdataquery1""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -281,15 +245,9 @@ test("OffHeapQuery-001-TC_025", Include) {
//To check query execution with IN operator without paranthesis
test("OffHeapQuery-001-TC_027", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id IN 9000,9005""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -304,15 +262,9 @@ test("OffHeapQuery-001-TC_028", Include) {
//To check query execution with IN operator with out specifying any field.
test("OffHeapQuery-001-TC_029", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where IN(1,2)""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -354,15 +306,9 @@ test("OffHeapQuery-001-TC_033", Include) {
//To check AND with using booleans in invalid syntax
test("OffHeapQuery-001-TC_034", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where AND true""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -386,15 +332,9 @@ test("OffHeapQuery-001-TC_036", Include) {
//To check AND using 0 and 1 treated as boolean values
test("OffHeapQuery-001-TC_037", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where true aNd 0""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -418,29 +358,17 @@ test("OffHeapQuery-001-TC_039", Include) {
//To check '='operator without Passing any value
test("OffHeapQuery-001-TC_040", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id=""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '='operator without Passing columnname and value.
test("OffHeapQuery-001-TC_041", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where =""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -455,15 +383,9 @@ test("OffHeapQuery-001-TC_042", Include) {
//To check '!='operator by keeping space between them
test("OffHeapQuery-001-TC_043", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id ! = 9001""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -478,29 +400,17 @@ test("OffHeapQuery-001-TC_044", Include) {
//To check '!='operator without providing any value
test("OffHeapQuery-001-TC_045", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id != """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '!='operator without providing any column name
test("OffHeapQuery-001-TC_046", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where != false""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -542,43 +452,25 @@ test("OffHeapQuery-001-TC_050", Include) {
//To check 'NOT' operator in nested way
test("OffHeapQuery-001-TC_051", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id NOT (NOT(true))""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'NOT' operator with parenthesis.
test("OffHeapQuery-001-TC_052", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id NOT ()""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'NOT' operator without condition.
test("OffHeapQuery-001-TC_053", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id NOT""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -593,29 +485,17 @@ test("OffHeapQuery-001-TC_054", Include) {
//To check '>' operator without specifying column
test("OffHeapQuery-001-TC_055", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where > 20""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '>' operator without specifying value
test("OffHeapQuery-001-TC_056", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id > """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -648,15 +528,9 @@ test("OffHeapQuery-001-TC_059", Include) {
//To check '<' operator without specifying column
test("OffHeapQuery-001-TC_060", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where < 5""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -680,29 +554,17 @@ test("OffHeapQuery-001-TC_062", Include) {
//To check '<=' operator without specifying column
test("OffHeapQuery-001-TC_063", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where <= 2""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '<=' operator without providing value
test("OffHeapQuery-001-TC_064", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id <= """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -717,29 +579,17 @@ test("OffHeapQuery-001-TC_065", Include) {
//To check '<=' operator adding space between'<' and '='
test("OffHeapQuery-001-TC_066", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id < = 9002""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'BETWEEN' operator without providing range
test("OffHeapQuery-001-TC_067", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where age between""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -799,29 +649,17 @@ test("OffHeapQuery-001-TC_073", Include) {
//To check 'IS NULL' without providing column
test("OffHeapQuery-001-TC_074", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where Is NulL""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'IS NOT NULL' without providing column
test("OffHeapQuery-001-TC_075", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where IS NOT NULL""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -854,29 +692,17 @@ test("OffHeapQuery-001-TC_078", Include) {
//To check Limit clause with where condition and no argument
test("OffHeapQuery-001-TC_079", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id=10987 limit""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check Limit clause with where condition and decimal argument
test("OffHeapQuery-001-TC_080", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id=10987 limit 0.0""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -927,15 +753,9 @@ test("OffHeapQuery-001-TC_085", Include) {
//To check Full join
test("OffHeapQuery-001-TC_086", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select uniqdataquery1.CUST_ID from uniqdataquery1 FULL JOIN uniqdataquery11 where CUST_ID""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1022,15 +842,9 @@ test("OffHeapQuery-001-TC_096", Include) {
//To check SORT using 'AND' on multiple column
test("OffHeapQuery-001-TC_097", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id > 10544 sort by cust_name desc and cust_id asc""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1054,15 +868,9 @@ test("OffHeapQuery-001-TC_099", Include) {
//To check average aggregate function with no arguments
test("OffHeapQuery-001-TC_100", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_id,avg() from uniqdataquery1 group by cust_id""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1077,15 +885,9 @@ test("OffHeapQuery-001-TC_101", Include) {
//To check nested average aggregate function
test("OffHeapQuery-001-TC_102", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_id,avg(count(cust_id)) from uniqdataquery1 group by cust_id""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1172,15 +974,9 @@ test("OffHeapQuery-001-TC_108", Include) {
//To check Order by without column name
test("OffHeapQuery-001-TC_112", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 order by ASC""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1222,15 +1018,9 @@ test("OffHeapQuery-001-TC_108", Include) {
//To check Using window without partition
test("OffHeapQuery-001-TC_117", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_name, sum(bigint_column1) OVER w from uniqdataquery1 WINDOW w""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1245,13 +1035,8 @@ test("OffHeapQuery-001-TC_108", Include) {
//To check Using ROLLUP without group by clause
test("OffHeapQuery-001-TC_119", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_name from uniqdataquery1 with ROLLUP""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table uniqdataquery1""").collect
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery2TestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery2TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery2TestCase.scala
index 10a9866..888070f 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery2TestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapQuery2TestCase.scala
@@ -44,15 +44,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check select query with limit as string
test("OffHeapQuery-002-TC_121", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 limit """"").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -112,57 +106,33 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check where clause with OR and no operand
test("OffHeapQuery-002-TC_128", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id > 1 OR """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check OR clause with LHS and RHS having no arguments
test("OffHeapQuery-002-TC_129", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where OR """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check OR clause with LHS having no arguments
test("OffHeapQuery-002-TC_130", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where OR cust_id > "1"""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check incorrect query
test("OffHeapQuery-002-TC_132", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id > 0 OR name """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -231,15 +201,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check select count and distinct query execution
test("OffHeapQuery-002-TC_140", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select count(cust_id),distinct(cust_name) from uniqdataquery2""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -281,15 +245,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check query execution with IN operator without paranthesis
test("OffHeapQuery-002-TC_146", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id IN 9000,9005""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -304,15 +262,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check query execution with IN operator with out specifying any field.
test("OffHeapQuery-002-TC_148", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where IN(1,2)""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -354,15 +306,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check AND with using booleans in invalid syntax
test("OffHeapQuery-002-TC_153", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where AND true""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -386,15 +332,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check AND using 0 and 1 treated as boolean values
test("OffHeapQuery-002-TC_156", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where true aNd 0""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -418,29 +358,17 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check '='operator without Passing any value
test("OffHeapQuery-002-TC_159", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id=""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '='operator without Passing columnname and value.
test("OffHeapQuery-002-TC_160", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where =""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -455,15 +383,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check '!='operator by keeping space between them
test("OffHeapQuery-002-TC_162", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id ! = 9001""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -478,29 +400,17 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check '!='operator without providing any value
test("OffHeapQuery-002-TC_164", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id != """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '!='operator without providing any column name
test("OffHeapQuery-002-TC_165", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where != false""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -542,43 +452,25 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check 'NOT' operator in nested way
test("OffHeapQuery-002-TC_170", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id NOT (NOT(true))""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'NOT' operator with parenthesis.
test("OffHeapQuery-002-TC_171", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id NOT ()""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'NOT' operator without condition.
test("OffHeapQuery-002-TC_172", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id NOT""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -593,29 +485,17 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check '>' operator without specifying column
test("OffHeapQuery-002-TC_174", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where > 20""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '>' operator without specifying value
test("OffHeapQuery-002-TC_175", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id > """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -648,15 +528,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check '<' operator without specifying column
test("OffHeapQuery-002-TC_179", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where < 5""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -680,29 +554,17 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check '<=' operator without specifying column
test("OffHeapQuery-002-TC_182", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where <= 2""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '<=' operator without providing value
test("OffHeapQuery-002-TC_183", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id <= """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -717,13 +579,8 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check '<=' operator adding space between'<' and '='
test("OffHeapQuery-002-TC_185", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id < = 9002""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
}
@@ -731,15 +588,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check 'BETWEEN' operator without providing range
test("OffHeapQuery-002-TC_186", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where age between""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -799,29 +650,17 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check 'IS NULL' without providing column
test("OffHeapQuery-002-TC_193", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where Is NulL""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'IS NOT NULL' without providing column
test("OffHeapQuery-002-TC_194", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where IS NOT NULL""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -854,29 +693,17 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check Limit clause with where condition and no argument
test("OffHeapQuery-002-TC_198", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id=10987 limit""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check Limit clause with where condition and decimal argument
test("OffHeapQuery-002-TC_199", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id=10987 limit 0.0""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -928,15 +755,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check Full join
test("OffHeapQuery-002-TC_205", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select uniqdataquery2.CUST_ID from uniqdataquery2 FULL JOIN uniqdataquery22 where CUST_ID""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1023,15 +844,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check SORT using 'AND' on multiple column
test("OffHeapQuery-002-TC_216", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 where cust_id > 10544 sort by cust_name desc and cust_id asc""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1055,15 +870,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check average aggregate function with no arguments
test("OffHeapQuery-002-TC_219", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_id,avg() from uniqdataquery2 group by cust_id""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1078,15 +887,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check nested average aggregate function
test("OffHeapQuery-002-TC_221", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_id,avg(count(cust_id)) from uniqdataquery2 group by cust_id""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1173,15 +976,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check Order by without column name
test("OffHeapQuery-002-TC_231", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery2 order by ASC""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1223,15 +1020,9 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check Using window without partition
test("OffHeapQuery-002-TC_236", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_name, sum(bigint_column1) OVER w from uniqdataquery2 WINDOW w""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1246,13 +1037,8 @@ class OffheapQuery2TestCase extends QueryTest with BeforeAndAfterAll {
//To check Using ROLLUP without group by clause
test("OffHeapQuery-002-TC_238", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_name from uniqdataquery2 with ROLLUP""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table uniqdataquery2""").collect
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort1TestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort1TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort1TestCase.scala
index 44287a2..b1cafee 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort1TestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort1TestCase.scala
@@ -74,13 +74,10 @@ class OffheapSort1TestCase extends QueryTest with BeforeAndAfterAll {
//To load data after setting offheap memory in carbon property file without folder path in load
test("OffHeapSort_001-TC_004", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE uniqdata13 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA into table uniqdata13 OPTIONS('DELIMITER'=',' , 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table uniqdata13""").collect
@@ -90,13 +87,10 @@ class OffheapSort1TestCase extends QueryTest with BeforeAndAfterAll {
//To load data after setting offheap memory in carbon property file without table_name in load
test("OffHeapSort_001-TC_005", Include) {
sql(s"""drop table if exists uniqdata14""").collect
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE uniqdata14 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/HeapVector/2000_UniqData.csv' into table OPTIONS('DELIMITER'=',' , 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata14""").collect
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort2TestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort2TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort2TestCase.scala
index b21ec20..21c74c9 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort2TestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/OffheapSort2TestCase.scala
@@ -70,14 +70,11 @@ class OffheapSort2TestCase extends QueryTest with BeforeAndAfterAll {
//To load data after setting offheap memory in carbon property file without folder path in load
test("OffHeapSort_002-TC_018", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata213""").collect
sql(s"""CREATE TABLE uniqdata213 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA into table uniqdata213 OPTIONS('DELIMITER'=',' , 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata213""").collect
@@ -87,14 +84,11 @@ class OffheapSort2TestCase extends QueryTest with BeforeAndAfterAll {
//To load data after setting offheap memory in carbon property file without table_name in load
test("OffHeapSort_002-TC_019", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata214""").collect
sql(s"""CREATE TABLE uniqdata214 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/HeapVector/2000_UniqData.csv' into table OPTIONS('DELIMITER'=',' , 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata214""").collect
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PartitionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PartitionTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PartitionTestCase.scala
index b89c353..31ec14e 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PartitionTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/PartitionTestCase.scala
@@ -31,12 +31,9 @@ class PartitionTestCase extends QueryTest with BeforeAndAfterAll {
//Verify exception if column in partitioned by is already specified in table schema
test("Partition-Local-sort_TC001", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
@@ -60,38 +57,31 @@ class PartitionTestCase extends QueryTest with BeforeAndAfterAll {
//Verify exception if List info is not given with List type partition
test("Partition-Local-sort_TC004", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
- //Verify exception if Partition type is not given
+ //exception should not be thrown if Partition type is not given
test("Partition-Local-sort_TC005", Include) {
try {
- sql(s"""drop table if exists uniqdata""").collect
+ sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('LIST_INFO'='1,2')""").collect
- assert(false)
+ sql(s"""drop table if exists uniqdata""").collect
} catch {
- case _ => assert(true)
+ case _ => assert(false)
}
- sql(s"""drop table if exists uniqdata""").collect
}
//Verify exception if Partition type is 'range' and LIST_INFO Is provided
test("Partition-Local-sort_TC006", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'LIST_INFO'='1,2')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
@@ -99,12 +89,9 @@ class PartitionTestCase extends QueryTest with BeforeAndAfterAll {
//Verify exception if Partition type is 'range' and NUM_PARTITIONS Is provided
test("Partition-Local-sort_TC007", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='RANGE', 'NUM_PARTITIONS'='1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
@@ -128,12 +115,9 @@ class PartitionTestCase extends QueryTest with BeforeAndAfterAll {
//Verify exception if Partition type is 'LIST' and NUM_PARTITIONS Is provided
test("Partition-Local-sort_TC010", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (DOJ int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='LIST', 'NUM_PARTITIONS'='1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
@@ -141,12 +125,9 @@ class PartitionTestCase extends QueryTest with BeforeAndAfterAll {
//Verify exception if Partition type is 'LIST' and RANGE_INFO Is provided
test("Partition-Local-sort_TC011", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='LIST', 'RANGE_INFO'='20160302,20150302')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
@@ -154,12 +135,9 @@ class PartitionTestCase extends QueryTest with BeforeAndAfterAll {
//Verify exception if datatype is not provided with partition column
test("Partition-Local-sort_TC012", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (DOJ)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='20160302,20150302')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
@@ -167,28 +145,23 @@ class PartitionTestCase extends QueryTest with BeforeAndAfterAll {
//Verify exception if a non existent file header is provided in partition
test("Partition-Local-sort_TC013", Include) {
- try {
- sql(s"""drop table if exists uniqdata""").collect
- sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='20160302,20150302')
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""").collect
+ sql(
+ s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='20160302,20150302')
LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOJ,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists uniqdata""").collect
+ sql(s"""drop table if exists uniqdata""").collect
}
//Verify exception if Partition By Is empty
test("Partition-Local-sort_TC014", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY ()STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='0,1')
""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
@@ -235,13 +208,10 @@ class PartitionTestCase extends QueryTest with BeforeAndAfterAll {
//Verify exception if 2 partition columns are provided
test("Partition-Local-sort_TC018", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""
CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (CUST_ID int , DOJ timestamp) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='0,1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
@@ -384,16 +354,13 @@ class PartitionTestCase extends QueryTest with BeforeAndAfterAll {
//Verify exception is thrown if partition column is dropped
test("Partition-Local-sort_TC029", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='LIST', 'LIST_INFO'='0,1')
alter table uniqdata drop columns(CUST_ID)
""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists uniqdata""").collect
}
[35/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 7f735c2..fd92fc1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -35,11 +35,10 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -72,7 +71,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
}
ifDefaultValueMatchesFilter();
- if (isDimensionPresentInCurrentBlock[0] == true) {
+ if (isDimensionPresentInCurrentBlock[0]) {
isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
&& dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
}
@@ -119,11 +118,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
boolean isScanRequired = false;
if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
- minValue = blockMinValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ minValue = blockMinValue[measureChunkIndex[0] + lastDimensionColOrdinal];
isScanRequired =
isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
- minValue = blockMinValue[dimensionBlocksIndex[0]];
+ minValue = blockMinValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(minValue, filterRangeValues);
}
} else {
@@ -169,67 +168,69 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
- } else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ } else {
+ int chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], this.msrFilterRangeValues,
msrColEvalutorInfoList.get(0).getType())) {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
}
- return null;
}
private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage,
@@ -262,7 +263,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return bitSet;
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
byte[] defaultValue = null;
if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
@@ -271,7 +272,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
dimColEvaluatorInfoList.get(0).getDimension().getDataType());
int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
- segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
+ segmentProperties.getDimensions().get(dimensionChunkIndex[0]);
if (currentBlockDimension.isSortColumn()) {
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
this.segmentProperties.getSortColumnsGenerator());
@@ -282,14 +283,14 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
}
BitSet bitSet = null;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows,
+ if (dimensionColumnPage.isExplicitSorted()) {
+ bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows,
defaultValue);
} else {
- bitSet = setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows, defaultValue);
+ bitSet = setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows, defaultValue);
}
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet,
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet,
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
}
return bitSet;
@@ -301,12 +302,12 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows,
+ DimensionColumnPage dimensionColumnPage, int numerOfRows,
byte[] defaultValue) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
@@ -318,7 +319,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
//find the number of default values to skip the null value in case of direct dictionary
if (null != defaultValue) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
defaultValue, true);
if (start < 0) {
skip = -(start + 1);
@@ -336,7 +337,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
for (int i = 0; i < filterValues.length; i++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i], false);
if (start >= 0) {
// Logic will handle the case where the range filter member is not present in block
@@ -344,7 +345,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
// set inorder to apply filters. this is Lesser than filter so the range will be taken
// from the prev element which is Lesser than filter member.
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[i]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[i]);
}
if (start < 0) {
start = -(start + 1);
@@ -355,14 +356,14 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
// will be pointing to the next consecutive position. So compare it again and point to the
// previous value returned from getFirstIndexUsingBinarySearch.
if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
< 0) {
start = start - 1;
}
}
last = start;
for (int j = start; j >= skip; j--) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
last--;
}
startIndex = last;
@@ -379,11 +380,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
* be called if the column is sorted default so column index
* mapping will be present for accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows, byte[] defaultValue) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
@@ -396,7 +397,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
//find the number of default values to skip the null value in case of direct dictionary
if (null != defaultValue) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, defaultValue, true);
if (start < 0) {
skip = -(start + 1);
@@ -413,11 +414,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
}
for (int k = 0; k < filterValues.length; k++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, filterValues[k], false);
if (start >= 0) {
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[k]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[k]);
}
if (start < 0) {
start = -(start + 1);
@@ -428,7 +429,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
// When negative value of start is returned from getFirstIndexUsingBinarySearch the Start
// will be pointing to the next consecutive position. So compare it again and point to the
// previous value returned from getFirstIndexUsingBinarySearch.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) < 0) {
+ if (ByteUtil.compare(filterValues[k], dimensionColumnPage.getChunkData(start)) < 0) {
start = start - 1;
}
}
@@ -445,7 +446,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
} else {
for (int k = 0; k < filterValues.length; k++) {
for (int i = 0; i < numerOfRows; i++) {
- if (ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValues[k]) < 0) {
+ if (ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValues[k]) < 0) {
bitSet.set(i);
}
}
@@ -454,21 +455,24 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- super.readBlocks(blockChunkHolder);
+ super.readColumnChunks(rawBlockletColumnChunks);
}
- int blockIndex = dimensionBlocksIndex[0];
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = dimensionChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex = measureBlocksIndex[0];
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = measureChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
index 92396ae..c5bea69 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
@@ -22,7 +22,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class TrueFilterExecutor implements FilterExecuter {
@@ -33,19 +33,20 @@ public class TrueFilterExecutor implements FilterExecuter {
* @return
* @throws FilterUnsupportedException
*/
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- int numberOfPages = blockChunkHolder.getDataBlock().numberOfPages();
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
BitSetGroup group = new BitSetGroup(numberOfPages);
for (int i = 0; i < numberOfPages; i++) {
BitSet set = new BitSet();
- set.flip(0, blockChunkHolder.getDataBlock().nodeSize());
+ set.flip(0, rawBlockletColumnChunks.getDataBlock().numRows());
group.setBitSet(set, i);
}
return group;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
return true;
}
@@ -66,9 +67,9 @@ public class TrueFilterExecutor implements FilterExecuter {
/**
* It just reads necessary block for filter executor, it does not uncompress the data.
*
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
*/
- public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) {
// do nothing
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
index 516447f..bc3b39c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
@@ -21,27 +21,29 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class ValueBasedFilterExecuterImpl implements FilterExecuter {
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
-
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
return new BitSetGroup(0);
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
throw new FilterUnsupportedException("Unsupported ValueBasedFilterExecuterImpl on row");
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
return new BitSet(1);
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java
deleted file mode 100644
index b3ed34e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.filter.intf;
-
-import org.apache.carbondata.core.scan.expression.Expression;
-
-public class FilterOptimizerBasic implements FilterOptimizer {
- @Override public Expression optimizeFilter() {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
index e5cb0ad..bbd33ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
@@ -19,14 +19,13 @@ package org.apache.carbondata.core.scan.filter.optimizer;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.RangeExpressionEvaluator;
-import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerImpl;
public class RangeFilterOptmizer extends FilterOptimizerImpl {
RangeExpressionEvaluator rangeExpEvaluator;
- public RangeFilterOptmizer(FilterOptimizer filterOptimizer, Expression filterExpression) {
+ public RangeFilterOptmizer(Expression filterExpression) {
super(filterExpression);
this.rangeExpEvaluator = new RangeExpressionEvaluator(filterExpression);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index bb66d18..996965d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -46,17 +46,13 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
protected boolean isIncludeFilter;
private DimColumnResolvedFilterInfo dimColResolvedFilterInfo;
private MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo;
- private AbsoluteTableIdentifier tableIdentifier;
- private boolean isMeasure;
public ConditionalFilterResolverImpl(Expression exp, boolean isExpressionResolve,
- boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier, boolean isMeasure) {
+ boolean isIncludeFilter, boolean isMeasure) {
this.exp = exp;
this.isExpressionResolve = isExpressionResolve;
this.isIncludeFilter = isIncludeFilter;
- this.tableIdentifier = tableIdentifier;
- this.isMeasure = isMeasure;
- if (isMeasure == false) {
+ if (!isMeasure) {
this.dimColResolvedFilterInfo = new DimColumnResolvedFilterInfo();
} else {
this.msrColResolvedFilterInfo = new MeasureColumnResolvedFilterInfo();
@@ -230,10 +226,6 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
return msrColResolvedFilterInfo;
}
- public AbsoluteTableIdentifier getTableIdentifier() {
- return tableIdentifier;
- }
-
/**
* method will calculates the start key based on the filter surrogates
*/
@@ -245,12 +237,6 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties,
setOfStartKeyByteArray);
}
-// else {
-// FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
-// segmentProperties, startKey, startKeyList);
-// FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties,
-// setOfStartKeyByteArray);
-// }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
index 33bafb1..c846c02 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
@@ -40,7 +40,7 @@ public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
public RowLevelFilterResolverImpl(Expression exp, boolean isExpressionResolve,
boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
- super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false);
+ super(exp, isExpressionResolve, isIncludeFilter, false);
dimColEvaluatorInfoList =
new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 0e7666c..e85e9d1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -61,7 +61,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
public RowLevelRangeFilterResolverImpl(Expression exp, boolean isExpressionResolve,
boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
- super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false);
+ super(exp, isExpressionResolve, isIncludeFilter, false);
dimColEvaluatorInfoList =
new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
index f8586ed..4165424 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
@@ -77,10 +77,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo
}
}
- public Map<CarbonMeasure, List<ColumnFilterInfo>> getMeasureResolvedFilterInstance() {
- return measureResolvedFilter;
- }
-
public ColumnFilterInfo getFilterValues() {
return resolvedFilterValueObj;
}
@@ -109,10 +105,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo
this.type = dataType;
}
- public boolean isMeasureExistsInCurrentSlice() {
- return true;
- }
-
public CarbonColumn getCarbonColumn() {
return carbonColumn;
}
@@ -125,10 +117,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo
return carbonMeasure;
}
- public boolean isMeasureExistsInCurrentSilce() {
- return isMeasureExistsInCurrentSilce;
- }
-
public void setMeasureExistsInCurrentSilce(boolean measureExistsInCurrentSilce) {
isMeasureExistsInCurrentSilce = measureExistsInCurrentSilce;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
index d403ef4..104f69e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
@@ -28,9 +28,9 @@ import org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolver
public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl {
public TrueConditionalResolverImpl(Expression exp, boolean isExpressionResolve,
- boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
+ boolean isIncludeFilter) {
- super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false);
+ super(exp, isExpressionResolve, isIncludeFilter, false);
}
@Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier,
@@ -47,16 +47,4 @@ public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl {
return FilterExecuterType.TRUE;
}
- /**
- * Method will the read filter expression corresponding to the resolver.
- * This method is required in row level executer inorder to evaluate the filter
- * expression against spark, as mentioned above row level is a special type
- * filter resolver.
- *
- * @return Expression
- */
- public Expression getFilterExpresion() {
- return exp;
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
index c90816a..b1e5a7d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
import java.io.IOException;
import java.util.Collections;
-import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
@@ -47,7 +46,6 @@ public class RangeDictionaryColumnVisitor extends DictionaryColumnVisitor
if (visitableObj instanceof DimColumnResolvedFilterInfo) {
DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
ColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal;
resolvedFilterObject = FilterUtil
.getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(),
metadata.getColumnExpression(), metadata.isIncludeFilter(),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java b/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java
deleted file mode 100644
index 5a5b042..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.model;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.scan.expression.Expression;
-
-/**
- * This class contains all the logical information about the query like dimensions,measures,
- * sort order, topN etc..
- */
-public class CarbonQueryPlan implements Serializable {
- /**
- *
- */
- private static final long serialVersionUID = -9036044826928017164L;
-
- /**
- * Database name
- */
- private String databaseName;
-
- /**
- * Table name
- */
- private String tableName;
-
- /**
- * List of dimensions.
- * Ex : select employee_name,department_name,sum(salary) from employee, then employee_name
- * and department_name are dimensions
- * If there is no dimensions asked in query then it would be remained as empty.
- */
- private List<QueryDimension> dimensions =
- new ArrayList<QueryDimension>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- /**
- * List of measures.
- * Ex : select employee_name,department_name,sum(salary) from employee, then sum(salary)
- * would be measure.
- * If there is no dimensions asked in query then it would be remained as empty.
- */
- private List<QueryMeasure> measures =
- new ArrayList<QueryMeasure>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- /**
- * expression
- */
- private Expression expression;
-
- /**
- * queryId
- */
- private String queryId;
-
- /**
- * If it is raw detail query, no need to aggregate in backend. And it returns with dictionary data
- * with out decoding.
- */
- private boolean rawDetailQuery;
-
- /**
- * Constructor created with database name and table name.
- *
- * @param databaseName
- * @param tableName
- */
- public CarbonQueryPlan(String databaseName, String tableName) {
- this.tableName = tableName;
- this.databaseName = databaseName;
- }
-
- /**
- * @return the dimensions
- */
- public List<QueryDimension> getDimensions() {
- return dimensions;
- }
-
- public void addDimension(QueryDimension dimension) {
- this.dimensions.add(dimension);
- }
-
- /**
- * @return the measures
- */
- public List<QueryMeasure> getMeasures() {
- return measures;
- }
-
- public void addMeasure(QueryMeasure measure) {
- this.measures.add(measure);
- }
-
- public Expression getFilterExpression() {
- return expression;
- }
-
- public void setFilterExpression(Expression expression) {
- this.expression = expression;
- }
-
- /**
- * @return the databaseName
- */
- public String getDatabaseName() {
- return databaseName;
- }
-
- /**
- * @return the tableName
- */
- public String getTableName() {
- return tableName;
- }
-
- public String getQueryId() {
- return queryId;
- }
-
- public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- public boolean isRawDetailQuery() {
- return rawDetailQuery;
- }
-
- public void setRawDetailQuery(boolean rawDetailQuery) {
- this.rawDetailQuery = rawDetailQuery;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java
new file mode 100644
index 0000000..0768221
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.model;
+
+/**
+ * Contains information for a column for projection
+ */
+public class ProjectionColumn {
+
+ /**
+ * name of the column
+ */
+ protected String columnName;
+
+ /**
+ * query order in which result of the query will be send
+ */
+ private int projectionOrdinal;
+
+ ProjectionColumn(String columnName) {
+ this.columnName = columnName;
+ }
+
+ /**
+ * @return the columnName
+ */
+ public String getColumnName() {
+ return columnName;
+ }
+
+ /**
+ * @return the projectionOrdinal
+ */
+ public int getOrdinal() {
+ return projectionOrdinal;
+ }
+
+ /**
+ * @param projectionOrdinal the projectionOrdinal to set
+ */
+ public void setOrdinal(int projectionOrdinal) {
+ this.projectionOrdinal = projectionOrdinal;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java
new file mode 100644
index 0000000..806de85
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+
+/**
+ * dimension column which holds the information about the column
+ * this is done to avoid heavy object serialization
+ */
+public class ProjectionDimension extends ProjectionColumn {
+
+ /**
+ * actual dimension column
+ */
+ private CarbonDimension dimension;
+
+ public ProjectionDimension(CarbonDimension dimension) {
+ super(dimension.getColName());
+ this.dimension = dimension;
+ }
+
+ /**
+ * @return the dimension
+ */
+ public CarbonDimension getDimension() {
+ return dimension;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java
new file mode 100644
index 0000000..93d9c82
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+
+/**
+ * query plan measure, this class will holds the information
+ * about measure present in the query, this is done to avoid the serialization
+ * of the heavy object
+ */
+public class ProjectionMeasure extends ProjectionColumn {
+
+ /**
+ * actual carbon measure object
+ */
+ private CarbonMeasure measure;
+
+ public ProjectionMeasure(CarbonMeasure measure) {
+ super(measure.getColName());
+ this.measure = measure;
+ }
+
+ /**
+ * @return the measure
+ */
+ public CarbonMeasure getMeasure() {
+ return measure;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java
deleted file mode 100644
index 1336b87..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.model;
-
-import java.io.Serializable;
-
-/**
- * query column which will have information about column
- */
-public class QueryColumn implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = -4222306600480181084L;
-
- /**
- * name of the column
- */
- protected String columnName;
-
- /**
- * query order in which result of the query will be send
- */
- private int queryOrder;
-
- public QueryColumn(String columnName) {
- this.columnName = columnName;
- }
-
- /**
- * @return the columnName
- */
- public String getColumnName() {
- return columnName;
- }
-
- /**
- * @return the queryOrder
- */
- public int getQueryOrder() {
- return queryOrder;
- }
-
- /**
- * @param queryOrder the queryOrder to set
- */
- public void setQueryOrder(int queryOrder) {
- this.queryOrder = queryOrder;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java
deleted file mode 100644
index e8f07a7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.model;
-
-import java.io.Serializable;
-
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-
-/**
- * query plan dimension which will holds the information about the query plan dimension
- * this is done to avoid heavy object serialization
- */
-public class QueryDimension extends QueryColumn implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = -8492704093776645651L;
- /**
- * actual dimension column
- */
- private transient CarbonDimension dimension;
-
- public QueryDimension(String columnName) {
- super(columnName);
- }
-
- /**
- * @return the dimension
- */
- public CarbonDimension getDimension() {
- return dimension;
- }
-
- /**
- * @param dimension the dimension to set
- */
- public void setDimension(CarbonDimension dimension) {
- this.dimension = dimension;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java
deleted file mode 100644
index 73a8ad5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.model;
-
-import java.io.Serializable;
-
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-
-/**
- * query plan measure, this class will holds the information
- * about measure present in the query, this is done to avoid the serialization
- * of the heavy object
- */
-public class QueryMeasure extends QueryColumn implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = 1035512411375495414L;
-
- /**
- * actual carbon measure object
- */
- private transient CarbonMeasure measure;
-
- public QueryMeasure(String columnName) {
- super(columnName);
- }
-
- /**
- * @return the measure
- */
- public CarbonMeasure getMeasure() {
- return measure;
- }
-
- /**
- * @param measure the measure to set
- */
- public void setMeasure(CarbonMeasure measure) {
- this.measure = measure;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 655ed5e..4e3665f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.core.scan.model;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -46,25 +45,17 @@ import org.apache.carbondata.core.util.DataTypeConverter;
* about the query, This will be sent from driver to executor '
* This will be refereed to executing the query.
*/
-public class QueryModel implements Serializable {
+public class QueryModel {
/**
- * serialization version
- */
- private static final long serialVersionUID = -4674677234007089052L;
- /**
* this will hold the information about the dictionary dimension
* which to
*/
- public transient Map<String, Dictionary> columnToDictionaryMapping;
- /**
- * list of dimension selected for in query
- */
- private List<QueryDimension> queryDimension;
+ private transient Map<String, Dictionary> columnToDictionaryMapping;
/**
- * list of measure selected in query
+ * list of projection columns in query
*/
- private List<QueryMeasure> queryMeasures;
+ private QueryProjection projection;
/**
* query id
*/
@@ -79,10 +70,6 @@ public class QueryModel implements Serializable {
*/
private List<TableBlockInfo> tableBlockInfos;
/**
- * absolute table identifier
- */
- private AbsoluteTableIdentifier absoluteTableIdentifier;
- /**
* To handle most of the computation in query engines like spark and hive, carbon should give
* raw detailed records to it.
*/
@@ -117,43 +104,15 @@ public class QueryModel implements Serializable {
*/
private boolean readPageByPage;
- public QueryModel() {
+ private QueryModel(CarbonTable carbonTable) {
tableBlockInfos = new ArrayList<TableBlockInfo>();
- queryDimension = new ArrayList<QueryDimension>();
- queryMeasures = new ArrayList<QueryMeasure>();
invalidSegmentIds = new ArrayList<>();
+ this.table = carbonTable;
+ this.queryId = String.valueOf(System.nanoTime());
}
- public static QueryModel createModel(AbsoluteTableIdentifier absoluteTableIdentifier,
- CarbonQueryPlan queryPlan, CarbonTable carbonTable, DataTypeConverter converter) {
- QueryModel queryModel = new QueryModel();
- queryModel.setAbsoluteTableIdentifier(absoluteTableIdentifier);
-
- fillQueryModel(queryPlan, carbonTable, queryModel);
-
- queryModel.setForcedDetailRawQuery(queryPlan.isRawDetailQuery());
- queryModel.setQueryId(queryPlan.getQueryId());
- queryModel.setConverter(converter);
- return queryModel;
- }
-
- private static void fillQueryModel(CarbonQueryPlan queryPlan, CarbonTable carbonTable,
- QueryModel queryModel) {
- queryModel.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
- queryModel.setQueryDimension(queryPlan.getDimensions());
- queryModel.setQueryMeasures(queryPlan.getMeasures());
- if (null != queryPlan.getFilterExpression()) {
- boolean[] isFilterDimensions = new boolean[carbonTable.getDimensionOrdinalMax()];
- boolean[] isFilterMeasures =
- new boolean[carbonTable.getNumberOfMeasures(carbonTable.getTableName())];
- processFilterExpression(carbonTable, queryPlan.getFilterExpression(), isFilterDimensions,
- isFilterMeasures);
- queryModel.setIsFilterDimensions(isFilterDimensions);
- queryModel.setIsFilterMeasures(isFilterMeasures);
- }
- //TODO need to remove this code, and executor will load the table
- // from file metadata
- queryModel.setTable(carbonTable);
+ public static QueryModel newInstance(CarbonTable carbonTable) {
+ return new QueryModel(carbonTable);
}
public static void processFilterExpression(CarbonTable carbonTable, Expression filterExpression,
@@ -239,42 +198,26 @@ public class QueryModel implements Serializable {
*/
public CarbonColumn[] getProjectionColumns() {
CarbonColumn[] carbonColumns =
- new CarbonColumn[getQueryDimension().size() + getQueryMeasures().size()];
- for (QueryDimension dimension : getQueryDimension()) {
- carbonColumns[dimension.getQueryOrder()] = dimension.getDimension();
+ new CarbonColumn[getProjectionDimensions().size() + getProjectionMeasures().size()];
+ for (ProjectionDimension dimension : getProjectionDimensions()) {
+ carbonColumns[dimension.getOrdinal()] = dimension.getDimension();
}
- for (QueryMeasure msr : getQueryMeasures()) {
- carbonColumns[msr.getQueryOrder()] = msr.getMeasure();
+ for (ProjectionMeasure msr : getProjectionMeasures()) {
+ carbonColumns[msr.getOrdinal()] = msr.getMeasure();
}
return carbonColumns;
}
- /**
- * @return the queryDimension
- */
- public List<QueryDimension> getQueryDimension() {
- return queryDimension;
+ public void setProjection(QueryProjection projection) {
+ this.projection = projection;
}
- /**
- * @param queryDimension the queryDimension to set
- */
- public void setQueryDimension(List<QueryDimension> queryDimension) {
- this.queryDimension = queryDimension;
+ public List<ProjectionDimension> getProjectionDimensions() {
+ return projection.getDimensions();
}
- /**
- * @return the queryMeasures
- */
- public List<QueryMeasure> getQueryMeasures() {
- return queryMeasures;
- }
-
- /**
- * @param queryMeasures the queryMeasures to set
- */
- public void setQueryMeasures(List<QueryMeasure> queryMeasures) {
- this.queryMeasures = queryMeasures;
+ public List<ProjectionMeasure> getProjectionMeasures() {
+ return projection.getMeasures();
}
/**
@@ -285,13 +228,6 @@ public class QueryModel implements Serializable {
}
/**
- * @param queryId the queryId to set
- */
- public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- /**
* @return the tableBlockInfos
*/
public List<TableBlockInfo> getTableBlockInfos() {
@@ -320,14 +256,7 @@ public class QueryModel implements Serializable {
* @return the absoluteTableIdentifier
*/
public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
- return absoluteTableIdentifier;
- }
-
- /**
- * @param absoluteTableIdentifier the absoluteTableIdentifier to set
- */
- public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
+ return table.getAbsoluteTableIdentifier();
}
/**
@@ -337,13 +266,6 @@ public class QueryModel implements Serializable {
return table;
}
- /**
- * @param table the table to set
- */
- public void setTable(CarbonTable table) {
- this.table = table;
- }
-
public boolean isForcedDetailRawQuery() {
return forcedDetailRawQuery;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java
new file mode 100644
index 0000000..d4496e0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+
+/**
+ * Contains projection columns in the query
+ */
+public class QueryProjection {
+
+ /**
+ * List of dimensions.
+ * Ex : select employee_name,department_name,sum(salary) from employee, then employee_name
+ * and department_name are dimensions
+ * If there is no dimensions asked in query then it would be remained as empty.
+ */
+ private List<ProjectionDimension> dimensions =
+ new ArrayList<ProjectionDimension>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+
+ /**
+ * List of measures.
+ * Ex : select employee_name,department_name,sum(salary) from employee, then sum(salary)
+ * would be measure.
+ * If there is no dimensions asked in query then it would be remained as empty.
+ */
+ private List<ProjectionMeasure> measures =
+ new ArrayList<ProjectionMeasure>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+
+ /**
+ * Constructor created with database name and table name.
+ *
+ */
+ public QueryProjection() {
+ }
+
+ /**
+ * @return the dimensions
+ */
+ public List<ProjectionDimension> getDimensions() {
+ return dimensions;
+ }
+
+ public void addDimension(CarbonDimension dimension, int queryOrdinal) {
+ ProjectionDimension queryDimension = new ProjectionDimension(dimension);
+ queryDimension.setOrdinal(queryOrdinal);
+ this.dimensions.add(queryDimension);
+ }
+
+ /**
+ * @return the measures
+ */
+ public List<ProjectionMeasure> getMeasures() {
+ return measures;
+ }
+
+ public void addMeasure(CarbonMeasure measure, int queryOrdinal) {
+ ProjectionMeasure queryMeasure = new ProjectionMeasure(measure);
+ queryMeasure.setOrdinal(queryOrdinal);
+ this.measures.add(queryMeasure);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
deleted file mode 100644
index eb5e3f9..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.processor;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.scan.collector.ResultCollectorFactory;
-import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.core.scan.scanner.BlockletScanner;
-import org.apache.carbondata.core.scan.scanner.impl.FilterScanner;
-import org.apache.carbondata.core.scan.scanner.impl.NonFilterScanner;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.util.TaskMetricsMap;
-
-/**
- * This abstract class provides a skeletal implementation of the
- * Block iterator.
- */
-public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Object[]>> {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractDataBlockIterator.class.getName());
-
- /**
- * iterator which will be used to iterate over data blocks
- */
- protected CarbonIterator<DataRefNode> dataBlockIterator;
-
- /**
- * result collector which will be used to aggregate the scanned result
- */
- protected ScannedResultCollector scannerResultAggregator;
-
- /**
- * processor which will be used to process the block processing can be
- * filter processing or non filter processing
- */
- protected BlockletScanner blockletScanner;
-
- /**
- * batch size of result
- */
- protected int batchSize;
-
- protected ExecutorService executorService;
-
- private Future<AbstractScannedResult> future;
-
- private Future<BlocksChunkHolder> futureIo;
-
- protected AbstractScannedResult scannedResult;
-
- private BlockExecutionInfo blockExecutionInfo;
-
- private FileHolder fileReader;
-
- private AtomicBoolean nextBlock;
-
- private AtomicBoolean nextRead;
-
- public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
- int batchSize, QueryStatisticsModel queryStatisticsModel, ExecutorService executorService) {
- this.blockExecutionInfo = blockExecutionInfo;
- this.fileReader = fileReader;
- dataBlockIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
- blockExecutionInfo.getNumberOfBlockToScan());
- if (blockExecutionInfo.getFilterExecuterTree() != null) {
- blockletScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel);
- } else {
- blockletScanner = new NonFilterScanner(blockExecutionInfo, queryStatisticsModel);
- }
- this.scannerResultAggregator =
- ResultCollectorFactory.getScannedResultCollector(blockExecutionInfo);
- this.batchSize = batchSize;
- this.executorService = executorService;
- this.nextBlock = new AtomicBoolean(false);
- this.nextRead = new AtomicBoolean(false);
- }
-
- public boolean hasNext() {
- if (scannedResult != null && scannedResult.hasNext()) {
- return true;
- } else {
- if (null != scannedResult) {
- scannedResult.freeMemory();
- }
- return dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get();
- }
- }
-
- protected boolean updateScanner() {
- try {
- if (scannedResult != null && scannedResult.hasNext()) {
- return true;
- } else {
- scannedResult = getNextScannedResult();
- while (scannedResult != null) {
- if (scannedResult.hasNext()) {
- return true;
- }
- scannedResult = getNextScannedResult();
- }
- nextBlock.set(false);
- nextRead.set(false);
- return false;
- }
- } catch (Exception ex) {
- throw new RuntimeException(ex);
- }
- }
-
- private AbstractScannedResult getNextScannedResult() throws Exception {
- AbstractScannedResult result = null;
- if (blockExecutionInfo.isPrefetchBlocklet()) {
- if (dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get()) {
- if (future == null) {
- future = execute();
- }
- result = future.get();
- nextBlock.set(false);
- if (dataBlockIterator.hasNext() || nextRead.get()) {
- nextBlock.set(true);
- future = execute();
- }
- }
- } else {
- if (dataBlockIterator.hasNext()) {
- BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolder();
- if (blocksChunkHolder != null) {
- result = blockletScanner.scanBlocklet(blocksChunkHolder);
- }
- }
- }
- return result;
- }
-
- private BlocksChunkHolder getBlocksChunkHolder() throws IOException {
- BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolderInternal();
- while (blocksChunkHolder == null && dataBlockIterator.hasNext()) {
- blocksChunkHolder = getBlocksChunkHolderInternal();
- }
- return blocksChunkHolder;
- }
-
- private BlocksChunkHolder getBlocksChunkHolderInternal() throws IOException {
- BlocksChunkHolder blocksChunkHolder =
- new BlocksChunkHolder(blockExecutionInfo.getTotalNumberDimensionBlock(),
- blockExecutionInfo.getTotalNumberOfMeasureBlock(), fileReader);
- blocksChunkHolder.setDataBlock(dataBlockIterator.next());
- if (blocksChunkHolder.getDataBlock().getColumnsMaxValue() == null) {
- return blocksChunkHolder;
- }
- if (blockletScanner.isScanRequired(blocksChunkHolder)) {
- return blocksChunkHolder;
- }
- return null;
- }
-
- private Future<AbstractScannedResult> execute() {
- return executorService.submit(new Callable<AbstractScannedResult>() {
- @Override public AbstractScannedResult call() throws Exception {
- if (futureIo == null) {
- futureIo = executeRead();
- }
- BlocksChunkHolder blocksChunkHolder = futureIo.get();
- futureIo = null;
- nextRead.set(false);
- if (blocksChunkHolder != null) {
- if (dataBlockIterator.hasNext()) {
- nextRead.set(true);
- futureIo = executeRead();
- }
- return blockletScanner.scanBlocklet(blocksChunkHolder);
- }
- return null;
- }
- });
- }
-
- private Future<BlocksChunkHolder> executeRead() {
- return executorService.submit(new Callable<BlocksChunkHolder>() {
- @Override public BlocksChunkHolder call() throws Exception {
- try {
- TaskMetricsMap.getInstance().registerThreadCallback();
- if (dataBlockIterator.hasNext()) {
- BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolder();
- if (blocksChunkHolder != null) {
- blockletScanner.readBlocklet(blocksChunkHolder);
- return blocksChunkHolder;
- }
- }
- return null;
- } finally {
- // update read bytes metrics for this thread
- TaskMetricsMap.getInstance().updateReadBytes(Thread.currentThread().getId());
- }
- }
- });
- }
-
- public abstract void processNextBatch(CarbonColumnarBatch columnarBatch);
-
- /**
- * Close the resources
- */
- public void close() {
- // free the current scanned result
- if (null != scannedResult && !scannedResult.hasNext()) {
- scannedResult.freeMemory();
- }
- // free any pre-fetched memory if present
- if (null != future) {
- try {
- AbstractScannedResult abstractScannedResult = future.get();
- if (abstractScannedResult != null) {
- abstractScannedResult.freeMemory();
- }
- } catch (InterruptedException | ExecutionException e) {
- throw new RuntimeException(e);
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
index 3ea4e1d..f0d81ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
@@ -23,7 +23,7 @@ import org.apache.carbondata.core.datastore.DataRefNode;
/**
* Below class will be used to iterate over data block
*/
-public class BlockletIterator extends CarbonIterator<DataRefNode> {
+class BlockletIterator extends CarbonIterator<DataRefNode> {
/**
* data store block
*/
@@ -47,17 +47,18 @@ public class BlockletIterator extends CarbonIterator<DataRefNode> {
* Constructor
*
* @param datablock first data block
- * @param totalNumberOfBlocksToScan total number of blocks to be scanned
+ * @param totalNumberOfBlockletToScan total number of blocklets to be scanned
*/
- public BlockletIterator(DataRefNode datablock, long totalNumberOfBlocksToScan) {
+ BlockletIterator(DataRefNode datablock, long totalNumberOfBlockletToScan) {
this.datablock = datablock;
- this.totalNumberOfBlocksToScan = totalNumberOfBlocksToScan;
+ this.totalNumberOfBlocksToScan = totalNumberOfBlockletToScan;
}
/**
* is all the blocks assigned to this iterator has been processed
*/
- @Override public boolean hasNext() {
+ @Override
+ public boolean hasNext() {
return hasNext;
}
@@ -67,7 +68,8 @@ public class BlockletIterator extends CarbonIterator<DataRefNode> {
* @return next data block
*
*/
- @Override public DataRefNode next() {
+ @Override
+ public DataRefNode next() {
// get the current blocks
DataRefNode datablockTemp = datablock;
// store the next data block
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
deleted file mode 100644
index 60090d0..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.processor;
-
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.util.BitSetGroup;
-
-/**
- * Block chunk holder which will hold the dimension and
- * measure chunk
- */
-public class BlocksChunkHolder {
-
- /**
- * dimension column data chunk
- */
- private DimensionRawColumnChunk[] dimensionRawDataChunk;
-
- /**
- * measure column data chunk
- */
- private MeasureRawColumnChunk[] measureRawDataChunk;
-
- /**
- * file reader which will use to read the block from file
- */
- private FileHolder fileReader;
-
- /**
- * data block
- */
- private DataRefNode dataBlock;
-
- private BitSetGroup bitSetGroup;
-
- public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock) {
- dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
- measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock];
- }
-
- public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock,
- FileHolder fileReader) {
- dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
- measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock];
- this.fileReader = fileReader;
- }
-
- /**
- * @return the dimensionRawDataChunk
- */
- public DimensionRawColumnChunk[] getDimensionRawDataChunk() {
- return dimensionRawDataChunk;
- }
-
- /**
- * @param dimensionRawDataChunk the dimensionRawDataChunk to set
- */
- public void setDimensionRawDataChunk(DimensionRawColumnChunk[] dimensionRawDataChunk) {
- this.dimensionRawDataChunk = dimensionRawDataChunk;
- }
-
- /**
- * @return the measureRawDataChunk
- */
- public MeasureRawColumnChunk[] getMeasureRawDataChunk() {
- return measureRawDataChunk;
- }
-
- /**
- * @param measureRawDataChunk the measureRawDataChunk to set
- */
- public void setMeasureRawDataChunk(MeasureRawColumnChunk[] measureRawDataChunk) {
- this.measureRawDataChunk = measureRawDataChunk;
- }
-
- /**
- * @return the fileReader
- */
- public FileHolder getFileReader() {
- return fileReader;
- }
-
- /**
- * @param fileReader the fileReader to set
- */
- public void setFileReader(FileHolder fileReader) {
- this.fileReader = fileReader;
- }
-
- /**
- * @return the dataBlock
- */
- public DataRefNode getDataBlock() {
- return dataBlock;
- }
-
- /**
- * @param dataBlock the dataBlock to set
- */
- public void setDataBlock(DataRefNode dataBlock) {
- this.dataBlock = dataBlock;
- }
-
- /***
- * To reset the measure chunk and dimension chunk
- * array
- */
- public void reset() {
- for (int i = 0; i < measureRawDataChunk.length; i++) {
- this.measureRawDataChunk[i] = null;
- }
- for (int i = 0; i < dimensionRawDataChunk.length; i++) {
- this.dimensionRawDataChunk[i] = null;
- }
- }
-
- public BitSetGroup getBitSetGroup() {
- return bitSetGroup;
- }
-
- public void setBitSetGroup(BitSetGroup bitSetGroup) {
- this.bitSetGroup = bitSetGroup;
- }
-}
[27/50] [abbrv] carbondata git commit: [CARBONDATA-1968] Add external
table support
Posted by ra...@apache.org.
[CARBONDATA-1968] Add external table support
This PR adds support for creating external table with existing carbondata files, using Hive syntax.
CREATE EXTERNAL TABLE tableName STORED BY 'carbondata' LOCATION 'path'
This closes #1749
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/111c3821
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/111c3821
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/111c3821
Branch: refs/heads/fgdatamap
Commit: 111c3821557820241d1114d87eae2f7cd017e610
Parents: 952665a
Author: Jacky Li <ja...@qq.com>
Authored: Tue Jan 2 23:46:14 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Feb 1 00:15:26 2018 +0800
----------------------------------------------------------------------
.../core/metadata/schema/table/CarbonTable.java | 9 ++
.../createTable/TestCreateExternalTable.scala | 91 ++++++++++++++++++++
.../TestDataWithDicExcludeAndInclude.scala | 10 ---
.../command/table/CarbonDropTableCommand.scala | 5 +-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 64 +++++++++-----
5 files changed, 147 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/111c3821/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 4bb0d20..07989b2 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -785,6 +785,15 @@ public class CarbonTable implements Serializable {
&& !tableInfo.getParentRelationIdentifiers().isEmpty();
}
+ /**
+ * Return true if this is an external table (table with property "_external"="true", this is
+ * an internal table property set during table creation)
+ */
+ public boolean isExternalTable() {
+ String external = tableInfo.getFactTable().getTableProperties().get("_external");
+ return external != null && external.equalsIgnoreCase("true");
+ }
+
public long size() throws IOException {
Map<String, Long> dataIndexSize = CarbonUtil.calculateDataIndexSize(this);
Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/111c3821/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
new file mode 100644
index 0000000..67370eb
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.createTable
+
+import java.io.File
+
+import org.apache.spark.sql.{AnalysisException, CarbonEnv}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
+
+ var originDataPath: String = _
+
+ override def beforeAll(): Unit = {
+ sql("DROP TABLE IF EXISTS origin")
+ // create carbon table and insert data
+ sql("CREATE TABLE origin(key INT, value STRING) STORED BY 'carbondata'")
+ sql("INSERT INTO origin select 100,'spark'")
+ sql("INSERT INTO origin select 200,'hive'")
+ originDataPath = s"$storeLocation/origin"
+ }
+
+ override def afterAll(): Unit = {
+ sql("DROP TABLE IF EXISTS origin")
+ }
+
+ test("create external table with existing files") {
+ assert(new File(originDataPath).exists())
+ sql("DROP TABLE IF EXISTS source")
+
+ // create external table with existing files
+ sql(
+ s"""
+ |CREATE EXTERNAL TABLE source
+ |STORED BY 'carbondata'
+ |LOCATION '$storeLocation/origin'
+ """.stripMargin)
+ checkAnswer(sql("SELECT count(*) from source"), sql("SELECT count(*) from origin"))
+
+ val carbonTable = CarbonEnv.getCarbonTable(None, "source")(sqlContext.sparkSession)
+ assert(carbonTable.isExternalTable)
+
+ sql("DROP TABLE IF EXISTS source")
+
+ // DROP TABLE should not delete data
+ assert(new File(originDataPath).exists())
+ }
+
+ test("create external table with empty folder") {
+ val exception = intercept[AnalysisException] {
+ sql(
+ s"""
+ |CREATE EXTERNAL TABLE source
+ |STORED BY 'carbondata'
+ |LOCATION './nothing'
+ """.stripMargin)
+ }
+ assert(exception.getMessage().contains("Invalid table path provided"))
+ }
+
+ test("create external table with CTAS") {
+ val exception = intercept[AnalysisException] {
+ sql(
+ """
+ |CREATE EXTERNAL TABLE source
+ |STORED BY 'carbondata'
+ |LOCATION './nothing'
+ |AS
+ | SELECT * FROM origin
+ """.stripMargin)
+ }
+ assert(exception.getMessage().contains("Create external table as select"))
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/111c3821/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
index c788857..201da39 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
@@ -90,16 +90,6 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with BeforeA
)
}
- test("test create external table should fail") {
- assert(intercept[AnalysisException](
- sql(
- """
- | CREATE EXTERNAL TABLE t1 (id string, value int)
- | STORED BY 'carbondata'
- """.stripMargin)
- ).message.contains("Operation not allowed: CREATE EXTERNAL TABLE"))
- }
-
override def afterAll {
dropTable
CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/111c3821/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
index 9c0eb57..5e00914 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
@@ -144,7 +144,10 @@ case class CarbonDropTableCommand(
// delete the table folder
val tablePath = carbonTable.getTablePath
val fileType = FileFactory.getFileType(tablePath)
- if (FileFactory.isFileExist(tablePath, fileType)) {
+
+ // delete table data only if it is not external table
+ if (FileFactory.isFileExist(tablePath, fileType) &&
+ !carbonTable.isExternalTable) {
val file = FileFactory.getCarbonFile(tablePath, fileType)
CarbonUtil.deleteFoldersAndFilesSilent(file)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/111c3821/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index 4b77417..ad6d0c7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -32,6 +32,8 @@ import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.CarbonReflectionUtils
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.hadoop.util.SchemaReader
import org.apache.carbondata.spark.CarbonOption
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -164,9 +166,6 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
if (bucketSpecContext != null) {
operationNotAllowed("CREATE TABLE ... CLUSTERED BY", bucketSpecContext)
}
- if (external) {
- operationNotAllowed("CREATE EXTERNAL TABLE", tableHeader)
- }
val cols = Option(columns).toSeq.flatMap(visitColTypeList)
val properties = getPropertyKeyValues(tablePropertyList)
@@ -231,6 +230,10 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
operationNotAllowed(
"Schema may not be specified in a Create Table As Select (CTAS) statement", columns)
}
+ // external table is not allow
+ if (external) {
+ operationNotAllowed("Create external table as select", tableHeader)
+ }
fields = parser
.getFields(CarbonEnv.getInstance(sparkSession).carbonMetastore
.getSchemaFromUnresolvedRelation(sparkSession, Some(q).get))
@@ -242,29 +245,48 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
}
// validate tblProperties
val bucketFields = parser.getBucketFields(tableProperties, fields, options)
- // prepare table model of the collected tokens
- val tableModel: TableModel = parser.prepareTableModel(
- ifNotExists,
- convertDbNameToLowerCase(tableIdentifier.database),
- tableIdentifier.table.toLowerCase,
- fields,
- partitionFields,
- tableProperties,
- bucketFields,
- isAlterFlow = false,
- tableComment)
+ val tableInfo = if (external) {
+ // read table info from schema file in the provided table path
+ val identifier = AbsoluteTableIdentifier.from(
+ tablePath.get,
+ CarbonEnv.getDatabaseName(tableIdentifier.database)(sparkSession),
+ tableIdentifier.table)
+ val table = try {
+ SchemaReader.getTableInfo(identifier)
+ } catch {
+ case e: Throwable =>
+ operationNotAllowed(s"Invalid table path provided: ${tablePath.get} ", tableHeader)
+ }
+ // set "_external" property, so that DROP TABLE will not delete the data
+ table.getFactTable.getTableProperties.put("_external", "true")
+ table
+ } else {
+ // prepare table model of the collected tokens
+ val tableModel: TableModel = parser.prepareTableModel(
+ ifNotExists,
+ convertDbNameToLowerCase(tableIdentifier.database),
+ tableIdentifier.table.toLowerCase,
+ fields,
+ partitionFields,
+ tableProperties,
+ bucketFields,
+ isAlterFlow = false,
+ tableComment)
+ TableNewProcessor(tableModel)
+ }
selectQuery match {
case query@Some(q) =>
CarbonCreateTableAsSelectCommand(
- TableNewProcessor(tableModel),
- query.get,
- tableModel.ifNotExistsSet,
- tablePath)
+ tableInfo = tableInfo,
+ query = query.get,
+ ifNotExistsSet = ifNotExists,
+ tableLocation = tablePath)
case _ =>
- CarbonCreateTableCommand(TableNewProcessor(tableModel),
- tableModel.ifNotExistsSet,
- tablePath)
+ CarbonCreateTableCommand(
+ tableInfo = tableInfo,
+ ifNotExistsSet = ifNotExists,
+ tableLocation = tablePath)
}
}
[08/50] [abbrv] carbondata git commit: [CARBONDATA-1987] Make package
name and directory paths consistent;
remove duplicate file CarbonColumnValidator
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSqlConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSqlConf.scala b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSqlConf.scala
new file mode 100644
index 0000000..2128ffd
--- /dev/null
+++ b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSqlConf.scala
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.internal.SQLConf.buildConf
+
+import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * To initialize dynamic values default param
+ */
+class CarbonSQLConf(sparkSession: SparkSession) {
+
+ val carbonProperties = CarbonProperties.getInstance()
+
+ /**
+ * To initialize dynamic param defaults along with usage docs
+ */
+ def addDefaultCarbonParams(): Unit = {
+ val ENABLE_UNSAFE_SORT =
+ buildConf(CarbonCommonConstants.ENABLE_UNSAFE_SORT)
+ .doc("To enable/ disable unsafe sort.")
+ .booleanConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
+ val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
+ buildConf(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
+ .doc("To set carbon task distribution.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
+ val BAD_RECORDS_LOGGER_ENABLE =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
+ .doc("To enable/ disable carbon bad record logger.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants
+ .CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
+ val BAD_RECORDS_ACTION =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
+ .doc("To configure the bad records action.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
+ val IS_EMPTY_DATA_BAD_RECORD =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
+ .doc("Property to decide weather empty data to be considered bad/ good record.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT
+ .toBoolean)
+ val SORT_SCOPE =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE)
+ .doc("Property to specify sort scope.")
+ .stringConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
+ val BATCH_SORT_SIZE_INMB =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB)
+ .doc("Property to specify batch sort size in MB.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
+ val SINGLE_PASS =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS)
+ .doc("Property to enable/disable single_pass.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
+ val BAD_RECORD_PATH =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH)
+ .doc("Property to configure the bad record location.")
+ .stringConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ val GLOBAL_SORT_PARTITIONS =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS)
+ .doc("Property to configure the global sort partitions.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
+ CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
+ val DATEFORMAT =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
+ .doc("Property to configure data format for date type columns.")
+ .stringConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
+ val CARBON_INPUT_SEGMENTS = buildConf(
+ "carbon.input.segments.<database_name>.<table_name>")
+ .doc("Property to configure the list of segments to query.").stringConf
+ .createWithDefault(carbonProperties
+ .getProperty("carbon.input.segments.<database_name>.<table_name>", "*"))
+ }
+ /**
+ * to set the dynamic properties default values
+ */
+ def addDefaultCarbonSessionParams(): Unit = {
+ sparkSession.conf.set(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
+ sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
+ carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
+ CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cdfcd75..2399e26 100644
--- a/pom.xml
+++ b/pom.xml
@@ -494,6 +494,7 @@
<sourceDirectory>${basedir}/hadoop/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark2/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark2/src/main/java</sourceDirectory>
+ <sourceDirectory>${basedir}/integration/spark2/src/main/spark2.1</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common-test/src/main/scala</sourceDirectory>
@@ -503,7 +504,7 @@
<sourceDirectory>${basedir}/integration/presto/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/presto/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/streaming/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/streaming/src/main/java</sourceDirectory>
+ <sourceDirectory>${basedir}/streaming/src/main/scala</sourceDirectory>
</sourceDirectories>
</configuration>
</plugin>
@@ -534,7 +535,7 @@
<artifactId>coveralls-maven-plugin</artifactId>
<version>4.3.0</version>
<configuration>
- <repoToken>Q8mcBW8dw34wjNiPAMx0SFQidspIwHw8g</repoToken>
+ <repoToken>opPwqWW41vYppv6KISea3u1TJvE1ugJ5Y</repoToken>
<sourceEncoding>UTF-8</sourceEncoding>
<jacocoReports>
<jacocoReport>${basedir}/target/carbondata-coverage-report/carbondata-coverage-report.xml
@@ -546,6 +547,7 @@
<sourceDirectory>${basedir}/processing/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/hadoop/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark2/src/main/scala</sourceDirectory>
+ <sourceDirectory>${basedir}/integration/spark2/src/main/spark2.2</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark2/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common/src/main/java</sourceDirectory>
@@ -556,7 +558,7 @@
<sourceDirectory>${basedir}/integration/presto/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/presto/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/streaming/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/streaming/src/main/java</sourceDirectory>
+ <sourceDirectory>${basedir}/streaming/src/main/scala</sourceDirectory>
</sourceDirectories>
</configuration>
</plugin>
[14/50] [abbrv] carbondata git commit: [CARBONDATA-2106] Update
product document with page level reader property
Posted by ra...@apache.org.
[CARBONDATA-2106] Update product document with page level reader property
This closes #1892
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/43a37d7b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/43a37d7b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/43a37d7b
Branch: refs/heads/fgdatamap
Commit: 43a37d7bb09a56cb01b58661376b0ce964709143
Parents: 3c3f33d
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jan 31 11:42:22 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Jan 31 12:10:27 2018 +0530
----------------------------------------------------------------------
docs/configuration-parameters.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/43a37d7b/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 367d045..cfe24a5 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -119,6 +119,7 @@ This section provides the details of all the configurations required for CarbonD
| carbon.numberof.preserve.segments | 0 | If the user wants to preserve some number of segments from being compacted then he can set this property. Example: carbon.numberof.preserve.segments = 2 then 2 latest segments will always be excluded from the compaction. No segments will be preserved by default. |
| carbon.allowed.compaction.days | 0 | Compaction will merge the segments which are loaded with in the specific number of days configured. Example: If the configuration is 2, then the segments which are loaded in the time frame of 2 days only will get merged. Segments which are loaded 2 days apart will not be merged. This is disabled by default. |
| carbon.enable.auto.load.merge | false | To enable compaction while data loading. |
+|carbon.enable.page.level.reader.in.compaction|true|Enabling page level reader for compaction reduces the memory usage while compacting more number of segments. It allows reading only page by page instead of reading whole blocklet to memory.|
* **Query Configuration**
[12/50] [abbrv] carbondata git commit: [CARBONDATA-1964] Fixed bug to
set bad.records.action parameter using SET command
Posted by ra...@apache.org.
[CARBONDATA-1964] Fixed bug to set bad.records.action parameter using SET command
Fixed bug to set bad.records.action parameter using SET command
This closes #1819
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e3498201
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e3498201
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e3498201
Branch: refs/heads/fgdatamap
Commit: e3498201e084167d3268eb545c2a5ee34269705f
Parents: 8314ea2
Author: Geetika Gupta <ge...@knoldus.in>
Authored: Wed Jan 17 13:31:56 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Wed Jan 31 11:21:14 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/util/SessionParams.java | 4 +
.../sdv/generated/DataLoadingTestCase.scala | 2 -
.../badrecordloger/BadRecordActionTest.scala | 99 ++++++++++++++++++++
.../carbondata/spark/util/DataLoadingUtil.scala | 2 +-
4 files changed, 104 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e3498201/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
index afbd947..ddc7539 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION;
@@ -104,6 +105,9 @@ public class SessionParams implements Serializable {
throws InvalidConfigurationException {
boolean isValidConf = validateKeyValue(key, value);
if (isValidConf) {
+ if (key.equals(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)) {
+ value = value.toUpperCase();
+ }
if (doAuditing) {
LOGGER.audit("The key " + key + " with value " + value + " added in the session param");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e3498201/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
index 365547e..8ff47af 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingTestCase.scala
@@ -731,7 +731,6 @@ class DataLoadingTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"""drop table uniqdata""").collect
}
-
//Show loads--->Action=Fail--->Logger=False
test("BadRecord_Dataload_025", Include) {
dropTable("uniqdata")
@@ -745,7 +744,6 @@ class DataLoadingTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"""drop table uniqdata""").collect
}
-
//when insert into null data,query table output NullPointerException
test("HQ_DEFECT_2016111509706", Include) {
sql(s"""drop table IF EXISTS t_carbn01""").collect
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e3498201/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordActionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordActionTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordActionTest.scala
new file mode 100644
index 0000000..0249ddf
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordActionTest.scala
@@ -0,0 +1,99 @@
+package org.apache.carbondata.spark.testsuite.badrecordloger
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class BadRecordActionTest extends QueryTest with BeforeAndAfterAll {
+
+
+ val csvFilePath = s"$resourcesPath/badrecords/datasample.csv"
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ sql("drop table if exists sales")
+ }
+
+ test("test load for bad_record_action=force") {
+ sql("drop table if exists sales")
+ sql(
+ """CREATE TABLE IF NOT EXISTS sales(ID BigInt, date Timestamp, country String,
+ actual_price Double, Quantity int, sold_price Decimal(19,2)) STORED BY 'carbondata'""")
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE sales OPTIONS" +
+ "('bad_records_action'='force', 'DELIMITER'=" +
+ " ',', 'QUOTECHAR'= '\"')")
+ checkAnswer(sql("select count(*) from sales"),
+ Seq(Row(6)))
+
+ }
+
+ test("test load for bad_record_action=FORCE") {
+ sql("drop table if exists sales")
+ sql(
+ """CREATE TABLE IF NOT EXISTS sales(ID BigInt, date Timestamp, country String,
+ actual_price Double, Quantity int, sold_price Decimal(19,2)) STORED BY 'carbondata'""")
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE sales OPTIONS" +
+ "('bad_records_action'='FORCE', 'DELIMITER'=" +
+ " ',', 'QUOTECHAR'= '\"')")
+ checkAnswer(sql("select count(*) from sales"),
+ Seq(Row(6)))
+ }
+
+ test("test load for bad_record_action=fail") {
+ sql("drop table if exists sales")
+ sql(
+ """CREATE TABLE IF NOT EXISTS sales(ID BigInt, date Timestamp, country String,
+ actual_price Double, Quantity int, sold_price Decimal(19,2)) STORED BY 'carbondata'""")
+ intercept[Exception] {
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE sales OPTIONS" +
+ "('bad_records_action'='fail', 'DELIMITER'=" +
+ " ',', 'QUOTECHAR'= '\"')")
+ }
+ }
+
+ test("test load for bad_record_action=FAIL") {
+ sql("drop table if exists sales")
+ sql(
+ """CREATE TABLE IF NOT EXISTS sales(ID BigInt, date Timestamp, country String,
+ actual_price Double, Quantity int, sold_price Decimal(19,2)) STORED BY 'carbondata'""")
+ intercept[Exception] {
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE sales OPTIONS" +
+ "('bad_records_action'='FAIL', 'DELIMITER'=" +
+ " ',', 'QUOTECHAR'= '\"')")
+ }
+ }
+
+ test("test load for bad_record_action=ignore") {
+ sql("drop table if exists sales")
+ sql(
+ """CREATE TABLE IF NOT EXISTS sales(ID BigInt, date Timestamp, country String,
+ actual_price Double, Quantity int, sold_price Decimal(19,2)) STORED BY 'carbondata'""")
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE sales OPTIONS" +
+ "('bad_records_action'='ignore', 'DELIMITER'=" +
+ " ',', 'QUOTECHAR'= '\"')")
+ checkAnswer(sql("select count(*) from sales"),
+ Seq(Row(2)))
+ }
+
+ test("test load for bad_record_action=IGNORE") {
+ sql("drop table if exists sales")
+ sql(
+ """CREATE TABLE IF NOT EXISTS sales(ID BigInt, date Timestamp, country String,
+ actual_price Double, Quantity int, sold_price Decimal(19,2)) STORED BY 'carbondata'""")
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE sales OPTIONS" +
+ "('bad_records_action'='IGNORE', 'DELIMITER'=" +
+ " ',', 'QUOTECHAR'= '\"')")
+ checkAnswer(sql("select count(*) from sales"),
+ Seq(Row(2)))
+ }
+
+ override def afterAll() = {
+ sql("drop table if exists sales")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e3498201/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index b04a58e..5e9f7fe 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -286,7 +286,7 @@ object DataLoadingUtil {
TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName + "," + bad_records_logger_enable)
carbonLoadModel.setBadRecordsAction(
- TableOptionConstant.BAD_RECORDS_ACTION.getName + "," + bad_records_action)
+ TableOptionConstant.BAD_RECORDS_ACTION.getName + "," + bad_records_action.toUpperCase)
carbonLoadModel.setIsEmptyDataBadRecord(
DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," +
[26/50] [abbrv] carbondata git commit: [CARBONDATA-1992] Remove
partitionId in CarbonTablePath
Posted by ra...@apache.org.
[CARBONDATA-1992] Remove partitionId in CarbonTablePath
In CarbonTablePath, there is a deprecated partition id which is always 0, it should be removed to avoid confusion.
This closes #1765
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/952665a8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/952665a8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/952665a8
Branch: refs/heads/fgdatamap
Commit: 952665a8c1c52f28951463fef989333ae0e6d83e
Parents: 3dff273
Author: Jacky Li <ja...@qq.com>
Authored: Sat Jan 6 20:28:44 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Feb 1 00:15:23 2018 +0800
----------------------------------------------------------------------
.../core/metadata/PartitionMapFileStore.java | 2 +-
.../core/mutate/CarbonUpdateUtil.java | 8 +-
.../core/statusmanager/LoadMetadataDetails.java | 2 +
.../SegmentUpdateStatusManager.java | 8 +-
.../apache/carbondata/core/util/CarbonUtil.java | 6 +-
.../core/util/path/CarbonTablePath.java | 55 +++---
.../CarbonFormatDirectoryStructureTest.java | 4 +-
.../hadoop/api/CarbonTableInputFormat.java | 2 +-
.../streaming/CarbonStreamRecordWriter.java | 2 +-
.../hadoop/test/util/StoreCreator.java | 1 -
.../presto/util/CarbonDataStoreCreator.scala | 1 -
.../dataload/TestLoadDataGeneral.scala | 2 +-
.../InsertIntoCarbonTableTestCase.scala | 4 +-
.../dataload/TestBatchSortDataLoad.scala | 3 +-
.../dataload/TestDataLoadWithFileName.scala | 2 +-
.../dataload/TestGlobalSortDataLoad.scala | 4 +-
.../testsuite/datamap/TestDataMapCommand.scala | 34 ++--
.../TestDataLoadingForPartitionTable.scala | 3 +-
.../StandardPartitionTableCleanTestCase.scala | 2 +-
...andardPartitionTableCompactionTestCase.scala | 2 +-
.../StandardPartitionTableLoadingTestCase.scala | 4 +-
.../load/DataLoadProcessBuilderOnSpark.scala | 1 -
.../load/DataLoadProcessorStepOnSpark.scala | 2 +-
.../spark/rdd/AlterTableLoadPartitionRDD.scala | 169 +++++++++----------
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 11 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala | 25 ++-
.../org/apache/spark/util/PartitionUtils.scala | 5 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 5 +-
.../datasources/CarbonFileFormat.scala | 1 -
.../partition/TestAlterPartitionTable.scala | 2 +-
.../bucketing/TableBucketingTestCase.scala | 2 +
.../loading/CarbonDataLoadConfiguration.java | 10 --
.../loading/DataLoadProcessBuilder.java | 1 -
.../loading/TableProcessingOperations.java | 3 +-
.../loading/model/CarbonLoadModel.java | 72 +-------
.../sort/impl/ParallelReadMergeSorterImpl.java | 4 +-
...arallelReadMergeSorterWithBucketingImpl.java | 15 +-
.../UnsafeBatchParallelReadMergeSorterImpl.java | 7 +-
...arallelReadMergeSorterWithBucketingImpl.java | 21 +--
.../CarbonRowDataWriterProcessorStepImpl.java | 33 ++--
.../steps/DataWriterBatchProcessorStepImpl.java | 25 +--
.../steps/DataWriterProcessorStepImpl.java | 22 +--
.../processing/merger/CarbonDataMergerUtil.java | 6 +-
.../merger/CompactionResultSortProcessor.java | 4 +-
.../sort/sortdata/SortParameters.java | 16 +-
.../store/CarbonFactDataHandlerModel.java | 7 +-
.../util/CarbonDataProcessorUtil.java | 12 +-
.../processing/util/CarbonLoaderUtil.java | 12 +-
.../processing/util/DeleteLoadFolders.java | 7 +-
.../carbondata/processing/StoreCreator.java | 1 -
.../carbondata/streaming/StreamHandoffRDD.scala | 1 -
.../streaming/StreamSinkFactory.scala | 2 +-
.../streaming/CarbonAppendableStreamSink.scala | 8 +-
53 files changed, 278 insertions(+), 385 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index 1e9cbc4..43310fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -363,7 +363,7 @@ public class PartitionMapFileStore {
List<String> toBeDeletedIndexFiles = new ArrayList<>();
List<String> toBeDeletedDataFiles = new ArrayList<>();
// take the list of files from this segment.
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment.getLoadName());
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segment.getLoadName());
String partitionFilePath = getPartitionFilePath(segmentPath);
if (partitionFilePath != null) {
PartitionMapper partitionMapper = readPartitionMap(partitionFilePath);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 0e4eec7..4b99307 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -297,9 +297,7 @@ public class CarbonUpdateUtil {
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
- // as of now considering only partition 0.
- String partitionId = "0";
- String partitionDir = carbonTablePath.getPartitionDir(partitionId);
+ String partitionDir = carbonTablePath.getPartitionDir();
CarbonFile file =
FileFactory.getCarbonFile(partitionDir, FileFactory.getFileType(partitionDir));
if (!file.exists()) {
@@ -380,7 +378,7 @@ public class CarbonUpdateUtil {
}
public static long getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
- String segmentDirPath = tablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentDirPath = tablePath.getCarbonDataDirectoryPath(segmentId);
// scan all the carbondata files and get the latest task ID.
CarbonFile segment =
@@ -445,7 +443,7 @@ public class CarbonUpdateUtil {
|| segment.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
// take the list of files from this segment.
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment.getLoadName());
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segment.getLoadName());
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index 85602bc..73a665d 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -123,10 +123,12 @@ public class LoadMetadataDetails implements Serializable {
*/
private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
+ @Deprecated
public String getPartitionCount() {
return partitionCount;
}
+ @Deprecated
public void setPartitionCount(String partitionCount) {
this.partitionCount = partitionCount;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index e0e7b70..d4ef5c6 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -197,7 +197,7 @@ public class SegmentUpdateStatusManager {
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
String endTimeStamp = "";
String startTimeStamp = "";
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
for (LoadMetadataDetails eachSeg : segmentDetails) {
@@ -292,7 +292,7 @@ public class SegmentUpdateStatusManager {
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
String segment = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID);
- String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment);
+ String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segment);
String completeBlockName = CarbonTablePath.addDataPartPrefix(
CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCK_ID)
+ CarbonCommonConstants.FACT_FILE_EXT);
@@ -424,7 +424,7 @@ public class SegmentUpdateStatusManager {
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
@@ -867,7 +867,7 @@ public class SegmentUpdateStatusManager {
// filter out the fact files.
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 600b1c9..f04b28d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1082,7 +1082,7 @@ public final class CarbonUtil {
// geting the index file path
//TODO need to pass proper partition number when partiton will be supported
String carbonIndexFilePath = carbonTablePath
- .getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
+ .getCarbonIndexFilePath(taskId, tableBlockInfoList.get(0).getSegmentId(),
bucketNumber, CarbonTablePath.DataFileUtil
.getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
tableBlockInfoList.get(0).getVersion());
@@ -1325,7 +1325,7 @@ public final class CarbonUtil {
// geting the index file path
//TODO need to pass proper partition number when partiton will be supported
String carbonIndexFilePath = carbonTablePath
- .getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
+ .getCarbonIndexFilePath(taskId, tableBlockInfoList.get(0).getSegmentId(),
bucketNumber, CarbonTablePath.DataFileUtil
.getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
tableBlockInfoList.get(0).getVersion());
@@ -2278,7 +2278,7 @@ public final class CarbonUtil {
long carbonDataSize = 0L;
long carbonIndexSize = 0L;
HashMap<String, Long> dataAndIndexSize = new HashMap<String, Long>();
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
FileFactory.FileType fileType = FileFactory.getFileType(segmentPath);
switch (fileType) {
case HDFS:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 9e66657..36eaa56 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -255,15 +255,14 @@ public class CarbonTablePath extends Path {
/**
* Gets absolute path of data file
*
- * @param partitionId unique partition identifier
* @param segmentId unique partition identifier
* @param filePartNo data file part number
* @param factUpdateTimeStamp unique identifier to identify an update
* @return absolute path of data file stored in carbon data format
*/
- public String getCarbonDataFilePath(String partitionId, String segmentId, Integer filePartNo,
- Long taskNo, int batchNo, int bucketNumber, String factUpdateTimeStamp) {
- return getSegmentDir(partitionId, segmentId) + File.separator + getCarbonDataFileName(
+ public String getCarbonDataFilePath(String segmentId, Integer filePartNo, Long taskNo,
+ int batchNo, int bucketNumber, String factUpdateTimeStamp) {
+ return getSegmentDir(segmentId) + File.separator + getCarbonDataFileName(
filePartNo, taskNo, bucketNumber, batchNo, factUpdateTimeStamp);
}
@@ -272,13 +271,12 @@ public class CarbonTablePath extends Path {
* based on task id
*
* @param taskId task id of the file
- * @param partitionId partition number
* @param segmentId segment number
* @return full qualified carbon index path
*/
- public String getCarbonIndexFilePath(final String taskId, final String partitionId,
- final String segmentId, final String bucketNumber) {
- String segmentDir = getSegmentDir(partitionId, segmentId);
+ public String getCarbonIndexFilePath(final String taskId, final String segmentId,
+ final String bucketNumber) {
+ String segmentDir = getSegmentDir(segmentId);
CarbonFile carbonFile =
FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
@@ -294,9 +292,8 @@ public class CarbonTablePath extends Path {
if (files.length > 0) {
return files[0].getAbsolutePath();
} else {
- throw new RuntimeException("Missing Carbon index file for partition["
- + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId
- + "]");
+ throw new RuntimeException("Missing Carbon index file for Segment[" + segmentId + "], "
+ + "taskId[" + taskId + "]");
}
}
@@ -304,8 +301,6 @@ public class CarbonTablePath extends Path {
* Below method will be used to get the carbon index file path
* @param taskId
* task id
- * @param partitionId
- * partition id
* @param segmentId
* segment id
* @param bucketNumber
@@ -314,28 +309,27 @@ public class CarbonTablePath extends Path {
* timestamp
* @return carbon index file path
*/
- public String getCarbonIndexFilePath(String taskId, String partitionId, String segmentId,
- String bucketNumber, String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
+ public String getCarbonIndexFilePath(String taskId, String segmentId, String bucketNumber,
+ String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
switch (columnarFormatVersion) {
case V1:
case V2:
- return getCarbonIndexFilePath(taskId, partitionId, segmentId, bucketNumber);
+ return getCarbonIndexFilePath(taskId, segmentId, bucketNumber);
default:
- String segmentDir = getSegmentDir(partitionId, segmentId);
+ String segmentDir = getSegmentDir(segmentId);
return segmentDir + File.separator + getCarbonIndexFileName(taskId,
Integer.parseInt(bucketNumber), timeStamp);
}
}
- public String getCarbonIndexFilePath(String taskId, String partitionId, String segmentId,
- int batchNo, String bucketNumber, String timeStamp,
- ColumnarFormatVersion columnarFormatVersion) {
+ public String getCarbonIndexFilePath(String taskId, String segmentId, int batchNo,
+ String bucketNumber, String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
switch (columnarFormatVersion) {
case V1:
case V2:
- return getCarbonIndexFilePath(taskId, partitionId, segmentId, bucketNumber);
+ return getCarbonIndexFilePath(taskId, segmentId, bucketNumber);
default:
- String segmentDir = getSegmentDir(partitionId, segmentId);
+ String segmentDir = getSegmentDir(segmentId);
return segmentDir + File.separator + getCarbonIndexFileName(Long.parseLong(taskId),
Integer.parseInt(bucketNumber), batchNo, timeStamp);
}
@@ -349,12 +343,11 @@ public class CarbonTablePath extends Path {
/**
* Gets absolute path of data file
*
- * @param partitionId unique partition identifier
* @param segmentId unique partition identifier
* @return absolute path of data file stored in carbon data format
*/
- public String getCarbonDataDirectoryPath(String partitionId, String segmentId) {
- return getSegmentDir(partitionId, segmentId);
+ public String getCarbonDataDirectoryPath(String segmentId) {
+ return getSegmentDir(segmentId);
}
/**
@@ -392,12 +385,16 @@ public class CarbonTablePath extends Path {
return segmentDir + File.separator + getCarbonStreamIndexFileName();
}
- public String getSegmentDir(String partitionId, String segmentId) {
- return getPartitionDir(partitionId) + File.separator + SEGMENT_PREFIX + segmentId;
+ public String getSegmentDir(String segmentId) {
+ return getPartitionDir() + File.separator + SEGMENT_PREFIX + segmentId;
}
- public String getPartitionDir(String partitionId) {
- return getFactDir() + File.separator + PARTITION_PREFIX + partitionId;
+ // This partition is not used in any code logic, just keep backward compatibility
+ public static final String DEPRECATED_PATITION_ID = "0";
+
+ public String getPartitionDir() {
+ return getFactDir() + File.separator + PARTITION_PREFIX +
+ CarbonTablePath.DEPRECATED_PATITION_ID;
}
private String getMetaDataDir() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
index 5549806..a1ccab3 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
@@ -53,8 +53,8 @@ public class CarbonFormatDirectoryStructureTest {
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.dictmeta"));
assertTrue(carbonTablePath.getSortIndexFilePath("t1_c1").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.sortindex"));
- assertTrue(carbonTablePath.getCarbonDataFilePath("1", "2", 3, 4L, 0, 0, "999").replace("\\", "/")
- .equals(CARBON_STORE + "/d1/t1/Fact/Part1/Segment_2/part-3-4_batchno0-0-999.carbondata"));
+ assertTrue(carbonTablePath.getCarbonDataFilePath("2", 3, 4L, 0, 0, "999").replace("\\", "/")
+ .equals(CARBON_STORE + "/d1/t1/Fact/Part0/Segment_2/part-3-4_batchno0-0-999.carbondata"));
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index a1887f0..6f1e123 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -480,7 +480,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
for (String segmentId : streamSegments) {
- String segmentDir = tablePath.getSegmentDir("0", segmentId);
+ String segmentDir = tablePath.getSegmentDir(segmentId);
FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
if (FileFactory.isFileExist(segmentDir, fileType)) {
String indexName = CarbonTablePath.getCarbonStreamIndexFileName();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
index 364a6a6..3ef8afc 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
@@ -129,7 +129,7 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
CarbonTablePath tablePath =
CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier());
- segmentDir = tablePath.getSegmentDir("0", segmentId);
+ segmentDir = tablePath.getSegmentDir(segmentId);
fileName = CarbonTablePath.getCarbonDataFileName(0, taskNo, 0, 0, "0");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index fbf33d6..ac17c4e 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -154,7 +154,6 @@ public class StoreCreator {
loadModel.setCsvHeaderColumns(loadModel.getCsvHeader().split(","));
loadModel.setTaskNo("0");
loadModel.setSegmentId("0");
- loadModel.setPartitionId("0");
loadModel.setFactTimeStamp(System.currentTimeMillis());
loadModel.setMaxColumns("10");
return loadModel;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index 7b5c311..a41e738 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -135,7 +135,6 @@ object CarbonDataStoreCreator {
loadModel.setCsvHeaderColumns(loadModel.getCsvHeader.split(","))
loadModel.setTaskNo("0")
loadModel.setSegmentId("0")
- loadModel.setPartitionId("0")
loadModel.setFactTimeStamp(System.currentTimeMillis())
loadModel.setMaxColumns("15")
executeGraph(loadModel, storePath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index 09ca9e5..c84ae6b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -49,7 +49,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
tableName: String): Boolean = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(datbaseName, tableName)
val partitionPath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir("0")
+ .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir
val fileType: FileFactory.FileType = FileFactory.getFileType(partitionPath)
val carbonFile = FileFactory.getCarbonFile(partitionPath, fileType)
val segments: ArrayBuffer[String] = ArrayBuffer()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index d59f0b5..5cc4156 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -232,7 +232,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select count(*) from CarbonOverwrite"), sql("select count(*) from HiveOverwrite"))
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbonoverwrite")
val partitionPath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir("0")
+ .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir
val folder = new File(partitionPath)
assert(folder.isDirectory)
assert(folder.list().length == 1)
@@ -255,7 +255,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select count(*) from TCarbonSourceOverwrite"), sql("select count(*) from HiveOverwrite"))
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "tcarbonsourceoverwrite")
val partitionPath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir("0")
+ .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir
val folder = new File(partitionPath)
assert(folder.isDirectory)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
index 4af9d54..42ac4df 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
@@ -193,9 +193,8 @@ class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
CarbonCommonConstants.DATABASE_DEFAULT_NAME,
tableName
)
- val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getCarbonDataDirectoryPath("0", segmentNo)
+ val segmentDir = carbonTablePath.getCarbonDataDirectoryPath(segmentNo)
new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
index dae0962..db0a62c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
@@ -49,7 +49,7 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
val indexReader = new CarbonIndexFileReader()
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "test_table_v3")
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getCarbonDataDirectoryPath("0", "0")
+ val segmentDir = carbonTablePath.getCarbonDataDirectoryPath("0")
val carbonIndexPaths = new File(segmentDir)
.listFiles(new FilenameFilter {
override def accept(dir: File, name: String): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index 50a38f1..fe5e327 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -272,7 +272,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql(s"LOAD DATA LOCAL INPATH '$inputPath' INTO TABLE carbon_globalsort")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbon_globalsort")
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getSegmentDir("0", "0")
+ val segmentDir = carbonTablePath.getSegmentDir("0")
assertResult(Math.max(4, defaultParallelism) + 1)(new File(segmentDir).listFiles().length)
}
@@ -379,7 +379,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
private def getIndexFileCount(tableName: String, segmentNo: String = "0"): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", tableName)
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getCarbonDataDirectoryPath("0", segmentNo)
+ val segmentDir = carbonTablePath.getCarbonDataDirectoryPath(segmentNo)
new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index 146ad62..5170c43 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -217,19 +217,27 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
}
test("test if preaggregate load is successfull for hivemetastore") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
- .stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
+ sql("DROP TABLE IF EXISTS maintable")
+ sql(
+ """
+ | CREATE TABLE maintable(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(
+ s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
+
+ .stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
+ checkAnswer(sql(s"select * from maintable_preagg_sum"),
+ Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
+ } finally {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ }
}
test("test preaggregate load for decimal column for hivemetastore") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
index ed151bd..0a21aed 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
@@ -63,7 +63,7 @@ class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
@@ -87,6 +87,7 @@ class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| PARTITIONED BY (empno int)
+ |
| STORED BY 'org.apache.carbondata.format'
| TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3')
""".stripMargin)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
index 2b0dd09..5427981 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
@@ -53,7 +53,7 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
index 22ebd80..f4b6e0e 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
@@ -53,7 +53,7 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 31d2598..7fcfb29 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -68,7 +68,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
@@ -335,7 +335,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_mergeindexpartitionthree")
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", "0")
+ val segmentDir = tablePath.getCarbonDataDirectoryPath("0")
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val files = carbonFile.listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = CarbonTablePath.isCarbonIndexFile(file.getName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
index 781b484..455f445 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
@@ -72,7 +72,6 @@ object DataLoadProcessBuilderOnSpark {
.map(DataLoadProcessorStepOnSpark.toStringArrayRow(_, columnCount))
}
- model.setPartitionId("0")
val sc = sparkSession.sparkContext
val modelBroadcast = sc.broadcast(model)
val partialSuccessAccum = sc.accumulator(0, "Partial Success Accumulator")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 21de003..834c1a6 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -194,7 +194,7 @@ object DataLoadProcessorStepOnSpark {
dataWriter = new DataWriterProcessorStepImpl(conf)
- val dataHandlerModel = dataWriter.getDataHandlerModel(0)
+ val dataHandlerModel = dataWriter.getDataHandlerModel
var dataHandler: CarbonFactHandler = null
var rowsNotExist = true
while (rows.hasNext) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
index 35a8ea7..c01ebed 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
@@ -41,102 +41,95 @@ class AlterTableLoadPartitionRDD[K, V](alterPartitionModel: AlterPartitionModel,
identifier: AbsoluteTableIdentifier,
prev: RDD[Array[AnyRef]]) extends RDD[(K, V)](prev) {
- var storeLocation: String = null
- val carbonLoadModel = alterPartitionModel.carbonLoadModel
- val segmentId = alterPartitionModel.segmentId
- val oldPartitionIds = alterPartitionModel.oldPartitionIds
- val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val databaseName = carbonTable.getDatabaseName
- val factTableName = carbonTable.getTableName
- val partitionInfo = carbonTable.getPartitionInfo(factTableName)
+ var storeLocation: String = null
+ val carbonLoadModel = alterPartitionModel.carbonLoadModel
+ val segmentId = alterPartitionModel.segmentId
+ val oldPartitionIds = alterPartitionModel.oldPartitionIds
+ val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ val databaseName = carbonTable.getDatabaseName
+ val factTableName = carbonTable.getTableName
+ val partitionInfo = carbonTable.getPartitionInfo(factTableName)
- override protected def getPartitions: Array[Partition] = {
- val sc = alterPartitionModel.sqlContext.sparkContext
- sc.setLocalProperty("spark.scheduler.pool", "DDL")
- sc.setLocalProperty("spark.job.interruptOnCancel", "true")
- firstParent[Array[AnyRef]].partitions
- }
+ override protected def getPartitions: Array[Partition] = {
+ val sc = alterPartitionModel.sqlContext.sparkContext
+ sc.setLocalProperty("spark.scheduler.pool", "DDL")
+ sc.setLocalProperty("spark.job.interruptOnCancel", "true")
+ firstParent[Array[AnyRef]].partitions
+ }
- override def compute(split: Partition, context: TaskContext): Iterator[(K, V)] = {
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
- val rows = firstParent[Array[AnyRef]].iterator(split, context).toList.asJava
- val iter = new Iterator[(K, V)] {
- val partitionId = partitionInfo.getPartitionId(split.index)
- carbonLoadModel.setTaskNo(String.valueOf(partitionId))
- carbonLoadModel.setSegmentId(segmentId)
- carbonLoadModel.setPartitionId("0")
- val tempLocationKey = CarbonDataProcessorUtil
- .getTempStoreLocationKey(carbonLoadModel.getDatabaseName,
- carbonLoadModel.getTableName,
- segmentId,
- carbonLoadModel.getTaskNo,
- false,
- true)
- // this property is used to determine whether temp location for carbon is inside
- // container temp dir or is yarn application directory.
- val carbonUseLocalDir = CarbonProperties.getInstance()
- .getProperty("carbon.use.local.dir", "false")
+ override def compute(split: Partition, context: TaskContext): Iterator[(K, V)] = {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+ val rows = firstParent[Array[AnyRef]].iterator(split, context).toList.asJava
+ val iter = new Iterator[(K, V)] {
+ val partitionId = partitionInfo.getPartitionId(split.index)
+ carbonLoadModel.setTaskNo(String.valueOf(partitionId))
+ carbonLoadModel.setSegmentId(segmentId)
+ val tempLocationKey = CarbonDataProcessorUtil
+ .getTempStoreLocationKey(carbonLoadModel.getDatabaseName,
+ carbonLoadModel.getTableName,
+ segmentId,
+ carbonLoadModel.getTaskNo,
+ false,
+ true)
+ // this property is used to determine whether temp location for carbon is inside
+ // container temp dir or is yarn application directory.
+ val carbonUseLocalDir = CarbonProperties.getInstance()
+ .getProperty("carbon.use.local.dir", "false")
- if (carbonUseLocalDir.equalsIgnoreCase("true")) {
+ if (carbonUseLocalDir.equalsIgnoreCase("true")) {
- val storeLocations = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
- if (null != storeLocations && storeLocations.nonEmpty) {
- storeLocation = storeLocations(Random.nextInt(storeLocations.length))
- }
- if (storeLocation == null) {
- storeLocation = System.getProperty("java.io.tmpdir")
- }
- } else {
- storeLocation = System.getProperty("java.io.tmpdir")
- }
- storeLocation = storeLocation + '/' + System.nanoTime() + '/' + split.index
- CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation)
- LOGGER.info(s"Temp storeLocation taken is $storeLocation")
+ val storeLocations = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
+ if (null != storeLocations && storeLocations.nonEmpty) {
+ storeLocation = storeLocations(Random.nextInt(storeLocations.length))
+ }
+ if (storeLocation == null) {
+ storeLocation = System.getProperty("java.io.tmpdir")
+ }
+ } else {
+ storeLocation = System.getProperty("java.io.tmpdir")
+ }
+ storeLocation = storeLocation + '/' + System.nanoTime() + '/' + split.index
+ CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation)
+ LOGGER.info(s"Temp storeLocation taken is $storeLocation")
- val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(databaseName,
- factTableName,
- carbonLoadModel.getTaskNo,
- "0",
- segmentId,
- false,
- true
- )
+ val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ databaseName, factTableName, carbonLoadModel.getTaskNo, segmentId, false, true)
- val loadStatus = if (rows.isEmpty) {
- LOGGER.info("After repartition this split, NO target rows to write back.")
- true
- } else {
- val segmentProperties = PartitionUtils.getSegmentProperties(identifier,
- segmentId, partitionIds.toList, oldPartitionIds, partitionInfo, carbonTable)
- val processor = new RowResultProcessor(
- carbonTable,
- carbonLoadModel,
- segmentProperties,
- tempStoreLoc,
- bucketId)
- try {
- processor.execute(rows)
- } catch {
- case e: Exception =>
- sys.error(s"Exception when executing Row result processor ${e.getMessage}")
- } finally {
- TableProcessingOperations
- .deleteLocalDataLoadFolderLocation(carbonLoadModel, false, true)
- }
- }
+ val loadStatus = if (rows.isEmpty) {
+ LOGGER.info("After repartition this split, NO target rows to write back.")
+ true
+ } else {
+ val segmentProperties = PartitionUtils.getSegmentProperties(identifier,
+ segmentId, partitionIds.toList, oldPartitionIds, partitionInfo, carbonTable)
+ val processor = new RowResultProcessor(
+ carbonTable,
+ carbonLoadModel,
+ segmentProperties,
+ tempStoreLoc,
+ bucketId)
+ try {
+ processor.execute(rows)
+ } catch {
+ case e: Exception =>
+ sys.error(s"Exception when executing Row result processor ${ e.getMessage }")
+ } finally {
+ TableProcessingOperations
+ .deleteLocalDataLoadFolderLocation(carbonLoadModel, false, true)
+ }
+ }
- val loadResult = segmentId
- var finished = false
+ val loadResult = segmentId
+ var finished = false
- override def hasNext: Boolean = {
- !finished
- }
+ override def hasNext: Boolean = {
+ !finished
+ }
- override def next(): (K, V) = {
- finished = true
- result.getKey(loadResult, loadStatus)
- }
- }
- iter
+ override def next(): (K, V) = {
+ finished = true
+ result.getKey(loadResult, loadStatus)
+ }
}
+ iter
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index c482a92..a9f8667 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -203,16 +203,9 @@ class CarbonMergerRDD[K, V](
}
}
- val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(databaseName,
- factTableName,
- carbonLoadModel.getTaskNo,
- "0",
- mergeNumber,
- true,
- false
- )
+ val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ databaseName, factTableName, carbonLoadModel.getTaskNo, mergeNumber, true, false)
- carbonLoadModel.setPartitionId("0")
var processor: AbstractResultProcessor = null
if (restructuredBlockExists) {
processor = new CompactionResultSortProcessor(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 72d0484..1fa1689 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.compression.CompressorFactory
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.loading.{DataLoadExecutor, FailureCauses, TableProcessingOperations}
import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat, CSVRecordReaderIterator}
import org.apache.carbondata.processing.loading.exception.NoRetryException
@@ -129,7 +130,8 @@ class SparkPartitionLoader(model: CarbonLoadModel,
System.setProperty("carbon.properties.filepath",
System.getProperty("user.dir") + '/' + "conf" + '/' + "carbon.properties")
}
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance.initPartitonInfo(model.getPartitionId)
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance.initPartitonInfo(
+ CarbonTablePath.DEPRECATED_PATITION_ID)
CarbonProperties.getInstance().addProperty("carbon.is.columnar.storage", "true")
CarbonProperties.getInstance().addProperty("carbon.dimension.split.value.in.columnar", "1")
CarbonProperties.getInstance().addProperty("carbon.is.fullyfilled.bits", "true")
@@ -219,14 +221,13 @@ class NewCarbonDataLoadRDD[K, V](
override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
- var partitionID = "0"
val loadMetadataDetails = new LoadMetadataDetails()
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
var model: CarbonLoadModel = _
val uniqueLoadStatusId =
carbonLoadModel.getTableName + CarbonCommonConstants.UNDERSCORE + theSplit.index
try {
- loadMetadataDetails.setPartitionCount(partitionID)
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID)
loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS)
val preFetch = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
@@ -264,7 +265,7 @@ class NewCarbonDataLoadRDD[K, V](
// So print the data load statistics only in case of non failure case
if (SegmentStatus.LOAD_FAILURE != loadMetadataDetails.getSegmentStatus) {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance
- .printStatisticsInfo(model.getPartitionId)
+ .printStatisticsInfo(CarbonTablePath.DEPRECATED_PATITION_ID)
}
}
@@ -287,8 +288,8 @@ class NewCarbonDataLoadRDD[K, V](
val fileList: java.util.List[String] = new java.util.ArrayList[String](
CarbonCommonConstants.CONSTANT_SIZE_TEN)
CarbonQueryUtil.splitFilePath(carbonLoadModel.getFactFilePath, fileList, ",")
- model = carbonLoadModel.getCopyWithPartition(partitionID, fileList,
- carbonLoadModel.getCsvHeader, carbonLoadModel.getCsvDelimiter)
+ model = carbonLoadModel.getCopyWithPartition(
+ carbonLoadModel.getCsvHeader, carbonLoadModel.getCsvDelimiter)
StandardLogService.setThreadName(StandardLogService
.getPartitionID(model.getCarbonDataLoadSchema.getCarbonTable.getTableUniqueName)
, ThreadLocalTaskInfo.getCarbonTaskInfo.getTaskId + "")
@@ -351,7 +352,6 @@ class NewDataFrameLoaderRDD[K, V](
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
- val partitionID = "0"
val loadMetadataDetails = new LoadMetadataDetails()
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
val model: CarbonLoadModel = carbonLoadModel
@@ -359,9 +359,8 @@ class NewDataFrameLoaderRDD[K, V](
carbonLoadModel.getTableName + CarbonCommonConstants.UNDERSCORE + theSplit.index
try {
- loadMetadataDetails.setPartitionCount(partitionID)
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID)
loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS)
- carbonLoadModel.setPartitionId(partitionID)
carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
carbonLoadModel.setPreFetch(false)
@@ -406,7 +405,7 @@ class NewDataFrameLoaderRDD[K, V](
// So print the data load statistics only in case of non failure case
if (SegmentStatus.LOAD_FAILURE != loadMetadataDetails.getSegmentStatus) {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance
- .printStatisticsInfo(model.getPartitionId)
+ .printStatisticsInfo(CarbonTablePath.DEPRECATED_PATITION_ID)
}
}
var finished = false
@@ -542,7 +541,6 @@ class PartitionTableDataLoaderRDD[K, V](
override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
- val partitionID = "0"
val loadMetadataDetails = new LoadMetadataDetails()
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
val model: CarbonLoadModel = carbonLoadModel
@@ -552,9 +550,8 @@ class PartitionTableDataLoaderRDD[K, V](
carbonLoadModel.getTableName + CarbonCommonConstants.UNDERSCORE + theSplit.index
try {
- loadMetadataDetails.setPartitionCount(partitionID)
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID)
loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS)
- carbonLoadModel.setPartitionId(partitionID)
carbonLoadModel.setTaskNo(String.valueOf(partitionInfo.getPartitionId(theSplit.index)))
carbonLoadModel.setPreFetch(false)
val recordReaders = Array[CarbonIterator[Array[AnyRef]]] {
@@ -590,7 +587,7 @@ class PartitionTableDataLoaderRDD[K, V](
// So print the data load statistics only in case of non failure case
if (SegmentStatus.LOAD_FAILURE != loadMetadataDetails.getSegmentStatus) {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance
- .printStatisticsInfo(model.getPartitionId)
+ .printStatisticsInfo(CarbonTablePath.DEPRECATED_PATITION_ID)
}
}
var finished = false
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
index 0498b25..3c871db 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
@@ -190,8 +190,9 @@ object PartitionUtils {
val batchNo = CarbonTablePath.DataFileUtil.getBatchNoFromTaskNo(taskNo)
val taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskNo)
val bucketNumber = CarbonTablePath.DataFileUtil.getBucketNo(path)
- val indexFilePath = carbonTablePath.getCarbonIndexFilePath(String.valueOf(taskId), "0",
- segmentId, batchNo, String.valueOf(bucketNumber), timestamp, version)
+ val indexFilePath = carbonTablePath.getCarbonIndexFilePath(
+ String.valueOf(taskId), segmentId, batchNo, String.valueOf(bucketNumber),
+ timestamp, version)
// indexFilePath could be duplicated when multiple data file related to one index file
if (indexFilePath != null && !pathList.contains(indexFilePath)) {
pathList.add(indexFilePath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 8212e85..6f056a6 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -613,7 +613,6 @@ object CarbonDataRDDFactory {
override def getPartition(key: Any): Int = {
val segId = key.asInstanceOf[String]
- // partitionId
segmentIdIndex(segId) * parallelism + Random.nextInt(parallelism)
}
}
@@ -647,7 +646,6 @@ object CarbonDataRDDFactory {
val rddResult = new updateResultImpl()
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val resultIter = new Iterator[(String, (LoadMetadataDetails, ExecutionErrors))] {
- val partitionID = "0"
val loadMetadataDetails = new LoadMetadataDetails
val executionErrors = ExecutionErrors(FailureCauses.NONE, "")
var uniqueLoadStatusId = ""
@@ -658,10 +656,9 @@ object CarbonDataRDDFactory {
CarbonCommonConstants.UNDERSCORE +
(index + "_0")
- loadMetadataDetails.setPartitionCount(partitionID)
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID)
loadMetadataDetails.setLoadName(segId)
loadMetadataDetails.setSegmentStatus(SegmentStatus.LOAD_FAILURE)
- carbonLoadModel.setPartitionId(partitionID)
carbonLoadModel.setSegmentId(segId)
carbonLoadModel.setTaskNo(String.valueOf(index))
carbonLoadModel.setFactTimeStamp(updateModel.get.updatedTimeStamp)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index 99e5732..945f47f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -105,7 +105,6 @@ with Serializable {
model,
conf
)
- model.setPartitionId("0")
model.setUseOnePass(options.getOrElse("onepass", "false").toBoolean)
model.setDictionaryServerHost(options.getOrElse("dicthost", null))
model.setDictionaryServerPort(options.getOrElse("dictport", "-1").toInt)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index b5325ef..aadee81 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -858,7 +858,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
def getDataFiles(carbonTable: CarbonTable, segmentId: String): Array[CarbonFile] = {
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
index 102df39..9da7244 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
@@ -173,6 +173,7 @@ class TableBucketingTestCase extends Spark2QueryTest with BeforeAndAfterAll {
case s: ShuffleExchange => shuffleExists = true
}
assert(!shuffleExists, "shuffle should not exist on bucket tables")
+ sql("DROP TABLE bucketed_parquet_table")
}
test("test create table with bucket join of carbon table and non bucket parquet table") {
@@ -197,6 +198,7 @@ class TableBucketingTestCase extends Spark2QueryTest with BeforeAndAfterAll {
case s: ShuffleExchange => shuffleExists = true
}
assert(shuffleExists, "shuffle should exist on non bucket tables")
+ sql("DROP TABLE parquet_table")
}
test("test scalar subquery with equal") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
index 7b1ab9d..e291f41 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
@@ -40,8 +40,6 @@ public class CarbonDataLoadConfiguration {
private String[] header;
- private String partitionId;
-
private String segmentId;
private String taskNo;
@@ -189,14 +187,6 @@ public class CarbonDataLoadConfiguration {
this.tableIdentifier = tableIdentifier;
}
- public String getPartitionId() {
- return partitionId;
- }
-
- public void setPartitionId(String partitionId) {
- this.partitionId = partitionId;
- }
-
public String getSegmentId() {
return segmentId;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index f7eff81..cf045a4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -152,7 +152,6 @@ public final class DataLoadProcessBuilder {
configuration.setTableIdentifier(identifier);
configuration.setSchemaUpdatedTimeStamp(carbonTable.getTableLastUpdatedTime());
configuration.setHeader(loadModel.getCsvHeaderColumns());
- configuration.setPartitionId(loadModel.getPartitionId());
configuration.setSegmentId(loadModel.getSegmentId());
configuration.setTaskNo(loadModel.getTaskNo());
configuration.setDataLoadProperty(DataLoadProcessorConstants.COMPLEX_DELIMITERS,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java b/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
index e2be79c..a8db6c9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
@@ -63,8 +63,7 @@ public class TableProcessingOperations {
//delete folder which metadata no exist in tablestatus
for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
- final String partitionCount = i + "";
- String partitionPath = carbonTablePath.getPartitionDir(partitionCount);
+ String partitionPath = carbonTablePath.getPartitionDir();
FileFactory.FileType fileType = FileFactory.getFileType(partitionPath);
if (FileFactory.isFileExist(partitionPath, fileType)) {
CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index d41455f..fef2da6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -39,8 +39,6 @@ public class CarbonLoadModel implements Serializable {
private String colDictFilePath;
- private String partitionId;
-
private CarbonDataLoadSchema carbonDataLoadSchema;
private boolean aggLoadRequest;
@@ -356,55 +354,6 @@ public class CarbonLoadModel implements Serializable {
}
/**
- * get copy with partition
- *
- * @param uniqueId
- * @return
- */
- public CarbonLoadModel getCopyWithPartition(String uniqueId) {
- CarbonLoadModel copy = new CarbonLoadModel();
- copy.tableName = tableName;
- copy.factFilePath = factFilePath + '/' + uniqueId;
- copy.databaseName = databaseName;
- copy.partitionId = uniqueId;
- copy.aggLoadRequest = aggLoadRequest;
- copy.loadMetadataDetails = loadMetadataDetails;
- copy.isRetentionRequest = isRetentionRequest;
- copy.complexDelimiterLevel1 = complexDelimiterLevel1;
- copy.complexDelimiterLevel2 = complexDelimiterLevel2;
- copy.carbonDataLoadSchema = carbonDataLoadSchema;
- copy.blocksID = blocksID;
- copy.taskNo = taskNo;
- copy.factTimeStamp = factTimeStamp;
- copy.segmentId = segmentId;
- copy.serializationNullFormat = serializationNullFormat;
- copy.badRecordsLoggerEnable = badRecordsLoggerEnable;
- copy.badRecordsAction = badRecordsAction;
- copy.escapeChar = escapeChar;
- copy.quoteChar = quoteChar;
- copy.commentChar = commentChar;
- copy.timestampformat = timestampformat;
- copy.dateFormat = dateFormat;
- copy.defaultTimestampFormat = defaultTimestampFormat;
- copy.maxColumns = maxColumns;
- copy.tablePath = tablePath;
- copy.useOnePass = useOnePass;
- copy.dictionaryServerHost = dictionaryServerHost;
- copy.dictionaryServerPort = dictionaryServerPort;
- copy.dictionaryServerSecretKey = dictionaryServerSecretKey;
- copy.dictionaryEncryptServerSecure = dictionaryEncryptServerSecure;
- copy.dictionaryServiceProvider = dictionaryServiceProvider;
- copy.preFetch = preFetch;
- copy.isEmptyDataBadRecord = isEmptyDataBadRecord;
- copy.skipEmptyLine = skipEmptyLine;
- copy.sortScope = sortScope;
- copy.batchSortSizeInMb = batchSortSizeInMb;
- copy.badRecordsLocation = badRecordsLocation;
- copy.isAggLoadRequest = isAggLoadRequest;
- return copy;
- }
-
- /**
* Get copy with taskNo.
* Broadcast value is shared in process, so we need to copy it to make sure the value in each
* task independently.
@@ -416,7 +365,6 @@ public class CarbonLoadModel implements Serializable {
copy.tableName = tableName;
copy.factFilePath = factFilePath;
copy.databaseName = databaseName;
- copy.partitionId = partitionId;
copy.aggLoadRequest = aggLoadRequest;
copy.loadMetadataDetails = loadMetadataDetails;
copy.isRetentionRequest = isRetentionRequest;
@@ -460,19 +408,15 @@ public class CarbonLoadModel implements Serializable {
/**
* get CarbonLoadModel with partition
*
- * @param uniqueId
- * @param filesForPartition
* @param header
* @param delimiter
* @return
*/
- public CarbonLoadModel getCopyWithPartition(String uniqueId, List<String> filesForPartition,
- String header, String delimiter) {
+ public CarbonLoadModel getCopyWithPartition(String header, String delimiter) {
CarbonLoadModel copyObj = new CarbonLoadModel();
copyObj.tableName = tableName;
copyObj.factFilePath = null;
copyObj.databaseName = databaseName;
- copyObj.partitionId = uniqueId;
copyObj.aggLoadRequest = aggLoadRequest;
copyObj.loadMetadataDetails = loadMetadataDetails;
copyObj.isRetentionRequest = isRetentionRequest;
@@ -514,20 +458,6 @@ public class CarbonLoadModel implements Serializable {
}
/**
- * @return the partitionId
- */
- public String getPartitionId() {
- return partitionId;
- }
-
- /**
- * @param partitionId the partitionId to set
- */
- public void setPartitionId(String partitionId) {
- this.partitionId = partitionId;
- }
-
- /**
* @param tablePath The tablePath to set.
*/
public void setTablePath(String tablePath) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
index 6432d38..fcc88b5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
@@ -73,8 +73,8 @@ public class ParallelReadMergeSorterImpl extends AbstractMergeSorter {
String[] storeLocations =
CarbonDataProcessorUtil.getLocalDataFolderLocation(
sortParameters.getDatabaseName(), sortParameters.getTableName(),
- String.valueOf(sortParameters.getTaskNo()), sortParameters.getPartitionID(),
- sortParameters.getSegmentId() + "", false, false);
+ String.valueOf(sortParameters.getTaskNo()), sortParameters.getSegmentId(),
+ false, false);
// Set the data file location
String[] dataFolderLocations = CarbonDataProcessorUtil.arrayAppend(storeLocations,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
index c7030dd..b7452a7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
@@ -133,10 +133,10 @@ public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorte
}
private SingleThreadFinalSortFilesMerger getFinalMerger(String bucketId) {
- String[] storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(sortParameters.getDatabaseName(), sortParameters.getTableName(),
- String.valueOf(sortParameters.getTaskNo()), bucketId,
- sortParameters.getSegmentId() + "", false, false);
+ String[] storeLocation = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ sortParameters.getDatabaseName(), sortParameters.getTableName(),
+ String.valueOf(sortParameters.getTaskNo()), sortParameters.getSegmentId(),
+ false, false);
// Set the data file location
String[] dataFolderLocation = CarbonDataProcessorUtil.arrayAppend(storeLocation, File.separator,
CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
@@ -181,10 +181,9 @@ public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorte
}
private void setTempLocation(SortParameters parameters) {
- String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(parameters.getDatabaseName(),
- parameters.getTableName(), parameters.getTaskNo(),
- parameters.getPartitionID(), parameters.getSegmentId(), false, false);
+ String[] carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ parameters.getDatabaseName(), parameters.getTableName(), parameters.getTaskNo(),
+ parameters.getSegmentId(), false, false);
String[] tmpLocs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator,
CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
parameters.setTempFileLocation(tmpLocs);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
index c5579d9..ed3a55d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
@@ -219,10 +219,9 @@ public class UnsafeBatchParallelReadMergeSorterImpl extends AbstractMergeSorter
}
private void setTempLocation(SortParameters parameters) {
- String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(parameters.getDatabaseName(),
- parameters.getTableName(), parameters.getTaskNo(), batchCount + "",
- parameters.getSegmentId(), false, false);
+ String[] carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ parameters.getDatabaseName(), parameters.getTableName(), parameters.getTaskNo(),
+ parameters.getSegmentId(), false, false);
String[] tempDirs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
parameters.setTempFileLocation(tempDirs);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
index 3c48e4d..f605b22 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
@@ -119,18 +119,17 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
Iterator<CarbonRowBatch>[] batchIterator = new Iterator[bucketingInfo.getNumberOfBuckets()];
for (int i = 0; i < sortDataRows.length; i++) {
- batchIterator[i] =
- new MergedDataIterator(String.valueOf(i), batchSize, intermediateFileMergers[i]);
+ batchIterator[i] = new MergedDataIterator(batchSize, intermediateFileMergers[i]);
}
return batchIterator;
}
- private UnsafeSingleThreadFinalSortFilesMerger getFinalMerger(String bucketId) {
- String[] storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(sortParameters.getDatabaseName(), sortParameters.getTableName(),
- String.valueOf(sortParameters.getTaskNo()), bucketId,
- sortParameters.getSegmentId() + "", false, false);
+ private UnsafeSingleThreadFinalSortFilesMerger getFinalMerger() {
+ String[] storeLocation = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ sortParameters.getDatabaseName(), sortParameters.getTableName(),
+ String.valueOf(sortParameters.getTaskNo()), sortParameters.getSegmentId(),
+ false, false);
// Set the data file location
String[] dataFolderLocation = CarbonDataProcessorUtil.arrayAppend(storeLocation,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
@@ -173,7 +172,7 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
private void setTempLocation(SortParameters parameters) {
String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
.getLocalDataFolderLocation(parameters.getDatabaseName(), parameters.getTableName(),
- parameters.getTaskNo(), parameters.getPartitionID(), parameters.getSegmentId(),
+ parameters.getTaskNo(), parameters.getSegmentId(),
false, false);
String[] tmpLoc = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator,
CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
@@ -224,7 +223,6 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
private class MergedDataIterator extends CarbonIterator<CarbonRowBatch> {
- private String partitionId;
private int batchSize;
@@ -232,9 +230,8 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
private UnsafeIntermediateMerger intermediateMerger;
- public MergedDataIterator(String partitionId, int batchSize,
+ public MergedDataIterator(int batchSize,
UnsafeIntermediateMerger intermediateMerger) {
- this.partitionId = partitionId;
this.batchSize = batchSize;
this.intermediateMerger = intermediateMerger;
this.firstRow = true;
@@ -245,7 +242,7 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
@Override public boolean hasNext() {
if (firstRow) {
firstRow = false;
- finalMerger = getFinalMerger(partitionId);
+ finalMerger = getFinalMerger();
List<UnsafeCarbonRowPage> rowPages = intermediateMerger.getRowPages();
finalMerger.startFinalMerge(rowPages.toArray(new UnsafeCarbonRowPage[rowPages.size()]),
intermediateMerger.getMergedPages());
[43/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all
path construction through CarbonTablePath static method
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 44204d4..f5a90de 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -32,8 +32,10 @@ import org.apache.spark.sql.types.StructType
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.{FileFormat, SegmentStatus}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
@@ -189,7 +191,6 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
val identifier = new TableIdentifier("batch_table", Option("streaming"))
val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
var server: ServerSocket = null
try {
server = getServerSocket
@@ -197,7 +198,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
thread1.start()
// use thread pool to catch the exception of sink thread
val pool = Executors.newSingleThreadExecutor()
- val thread2 = createSocketStreamingThread(spark, server.getLocalPort, tablePath, identifier)
+ val thread2 = createSocketStreamingThread(spark, server.getLocalPort, carbonTable, identifier)
val future = pool.submit(thread2)
Thread.sleep(1000)
thread1.interrupt()
@@ -220,11 +221,10 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
val identifier = new TableIdentifier("stream_table_file", Option("streaming"))
val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
val csvDataDir = new File("target/csvdata").getCanonicalPath
// streaming ingest 10 rows
generateCSVDataFile(spark, idStart = 10, rowNums = 10, csvDataDir)
- val thread = createFileStreamingThread(spark, tablePath, csvDataDir, intervalSecond = 1,
+ val thread = createFileStreamingThread(spark, carbonTable, csvDataDir, intervalSecond = 1,
identifier)
thread.start()
Thread.sleep(2000)
@@ -646,12 +646,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
val identifier = new TableIdentifier("stream_table_drop", Option("streaming"))
val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
var server: ServerSocket = null
try {
server = getServerSocket
val thread1 = createWriteSocketThread(server, 2, 10, 3)
- val thread2 = createSocketStreamingThread(spark, server.getLocalPort, tablePath, identifier, "force", 5, 1024L * 200, false)
+ val thread2 = createSocketStreamingThread(spark, server.getLocalPort, carbonTable, identifier, "force", 5, 1024L * 200, false)
thread1.start()
thread2.start()
Thread.sleep(1000)
@@ -749,7 +748,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
def createSocketStreamingThread(
spark: SparkSession,
port: Int,
- tablePath: CarbonTablePath,
+ carbonTable: CarbonTable,
tableIdentifier: TableIdentifier,
badRecordAction: String = "force",
intervalSecond: Int = 2,
@@ -770,7 +769,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
qry = readSocketDF.writeStream
.format("carbondata")
.trigger(ProcessingTime(s"$intervalSecond seconds"))
- .option("checkpointLocation", tablePath.getStreamingCheckpointDir)
+ .option("checkpointLocation", CarbonTablePath.getStreamingCheckpointDir(carbonTable.getTablePath))
.option("bad_records_action", badRecordAction)
.option("dbName", tableIdentifier.database.get)
.option("tableName", tableIdentifier.table)
@@ -808,7 +807,6 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
val identifier = new TableIdentifier(tableName, Option("streaming"))
val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
var server: ServerSocket = null
try {
server = getServerSocket()
@@ -821,7 +819,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
val thread2 = createSocketStreamingThread(
spark = spark,
port = server.getLocalPort,
- tablePath = tablePath,
+ carbonTable = carbonTable,
tableIdentifier = identifier,
badRecordAction = badRecordAction,
intervalSecond = intervalOfIngest,
@@ -863,7 +861,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
def createFileStreamingThread(
spark: SparkSession,
- tablePath: CarbonTablePath,
+ carbonTable: CarbonTable,
csvDataDir: String,
intervalSecond: Int,
tableIdentifier: TableIdentifier): Thread = {
@@ -889,7 +887,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
qry = readSocketDF.writeStream
.format("carbondata")
.trigger(ProcessingTime(s"${ intervalSecond } seconds"))
- .option("checkpointLocation", tablePath.getStreamingCheckpointDir)
+ .option("checkpointLocation", CarbonTablePath.getStreamingCheckpointDir(carbonTable.getTablePath))
.option("dbName", tableIdentifier.database.get)
.option("tableName", tableIdentifier.table)
.start()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala
index 9a6efbe..97dc8ba 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala
@@ -92,7 +92,7 @@ class AlterTableRevertTestCase extends Spark2QueryTest with BeforeAndAfterAll {
}
val carbonTable = CarbonMetadata.getInstance.getCarbonTable("default", "reverttest")
- assert(new File(carbonTable.getMetaDataFilepath).listFiles().length < 6)
+ assert(new File(carbonTable.getMetadataPath).listFiles().length < 6)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java b/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
index a8db6c9..bbc3697 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -56,43 +55,39 @@ public class TableProcessingOperations {
*/
public static void deletePartialLoadDataIfExist(CarbonTable carbonTable,
final boolean isCompactionFlow) throws IOException {
- String metaDataLocation = carbonTable.getMetaDataFilepath();
+ String metaDataLocation = carbonTable.getMetadataPath();
final LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation);
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getTablePath(), carbonTable.getCarbonTableIdentifier());
//delete folder which metadata no exist in tablestatus
- for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
- String partitionPath = carbonTablePath.getPartitionDir();
- FileFactory.FileType fileType = FileFactory.getFileType(partitionPath);
- if (FileFactory.isFileExist(partitionPath, fileType)) {
- CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
- CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile path) {
- String segmentId =
- CarbonTablePath.DataPathUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
- boolean found = false;
- for (int j = 0; j < details.length; j++) {
- if (details[j].getLoadName().equals(segmentId)) {
- found = true;
- break;
- }
- }
- return !found;
- }
- });
- for (int k = 0; k < listFiles.length; k++) {
+ String partitionPath = CarbonTablePath.getPartitionDir(carbonTable.getTablePath());
+ FileFactory.FileType fileType = FileFactory.getFileType(partitionPath);
+ if (FileFactory.isFileExist(partitionPath, fileType)) {
+ CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
+ CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile path) {
String segmentId =
- CarbonTablePath.DataPathUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
- if (isCompactionFlow) {
- if (segmentId.contains(".")) {
- CarbonLoaderUtil.deleteStorePath(listFiles[k].getAbsolutePath());
- }
- } else {
- if (!segmentId.contains(".")) {
- CarbonLoaderUtil.deleteStorePath(listFiles[k].getAbsolutePath());
+ CarbonTablePath.DataFileUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
+ boolean found = false;
+ for (int j = 0; j < details.length; j++) {
+ if (details[j].getLoadName().equals(segmentId)) {
+ found = true;
+ break;
}
}
+ return !found;
+ }
+ });
+ for (int k = 0; k < listFiles.length; k++) {
+ String segmentId =
+ CarbonTablePath.DataFileUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
+ if (isCompactionFlow) {
+ if (segmentId.contains(".")) {
+ CarbonLoaderUtil.deleteStorePath(listFiles[k].getAbsolutePath());
+ }
+ } else {
+ if (!segmentId.contains(".")) {
+ CarbonLoaderUtil.deleteStorePath(listFiles[k].getAbsolutePath());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
index 4cd5014..193d192 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
@@ -34,8 +34,6 @@ import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.datatypes.ArrayDataType;
import org.apache.carbondata.processing.datatypes.GenericDataType;
@@ -105,12 +103,11 @@ public class FieldEncoderFactory {
ColumnIdentifier parentColumnIdentifier =
new ColumnIdentifier(parentColumnTableRelation.getColumnId(), null,
dataField.getColumn().getDataType());
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
AbsoluteTableIdentifier parentAbsoluteTableIdentifier =
AbsoluteTableIdentifier.from(
- CarbonUtil.getNewTablePath(carbonTablePath, parentTableIdentifier.getTableName()),
- parentTableIdentifier);
+ CarbonTablePath.getNewTablePath(
+ absoluteTableIdentifier.getTablePath(), parentTableIdentifier.getTableName()),
+ parentTableIdentifier);
identifier = new DictionaryColumnUniqueIdentifier(parentAbsoluteTableIdentifier,
parentColumnIdentifier, dataField.getColumn().getDataType());
return new DictionaryFieldConverterImpl(dataField, cache, parentAbsoluteTableIdentifier,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
index d3caa99..a08177a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/AbstractResultProcessor.java
@@ -19,10 +19,8 @@ package org.apache.carbondata.processing.merger;
import java.util.List;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.store.CarbonDataFileAttributes;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
@@ -42,13 +40,11 @@ public abstract class AbstractResultProcessor {
public abstract boolean execute(List<RawResultIterator> resultIteratorList);
protected void setDataFileAttributesInModel(CarbonLoadModel loadModel,
- CompactionType compactionType, CarbonTable carbonTable,
- CarbonFactDataHandlerModel carbonFactDataHandlerModel) {
+ CompactionType compactionType, CarbonFactDataHandlerModel carbonFactDataHandlerModel) {
CarbonDataFileAttributes carbonDataFileAttributes;
if (compactionType == CompactionType.IUD_UPDDEL_DELTA) {
long taskNo = CarbonUpdateUtil.getLatestTaskIdForSegment(loadModel.getSegmentId(),
- CarbonStorePath.getCarbonTablePath(loadModel.getTablePath(),
- carbonTable.getCarbonTableIdentifier()));
+ loadModel.getTablePath());
// Increase the Task Index as in IUD_UPDDEL_DELTA_COMPACTION the new file will
// be written in same segment. So the TaskNo should be incremented by 1 from max val.
long index = taskNo + 1;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index d796262..4fa4ff4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -263,7 +263,7 @@ public class CarbonCompactionUtil {
public static CarbonTable getNextTableToCompact(CarbonTable[] carbonTables,
List<CarbonTableIdentifier> skipList) {
for (CarbonTable ctable : carbonTables) {
- String metadataPath = ctable.getMetaDataFilepath();
+ String metadataPath = ctable.getMetadataPath();
// check for the compaction required file and at the same time exclude the tables which are
// present in the skip list.
if (CarbonCompactionUtil.isCompactionRequiredForTable(metadataPath) && !skipList
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 0eadc7f..c43dbf9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -31,7 +31,6 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails;
@@ -42,7 +41,6 @@ import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
@@ -167,15 +165,13 @@ public final class CarbonDataMergerUtil {
// End Timestamp.
// Table Update Status Metadata Update.
- AbsoluteTableIdentifier absoluteTableIdentifier =
+ AbsoluteTableIdentifier identifier =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
-
SegmentUpdateStatusManager segmentUpdateStatusManager =
- new SegmentUpdateStatusManager(absoluteTableIdentifier);
+ new SegmentUpdateStatusManager(identifier);
- SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
+ SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
ICarbonLock updateLock = segmentUpdateStatusManager.getTableUpdateStatusLock();
ICarbonLock statusLock = segmentStatusManager.getTableStatusLock();
@@ -222,7 +218,7 @@ public final class CarbonDataMergerUtil {
}
LoadMetadataDetails[] loadDetails =
- segmentStatusManager.readLoadMetadata(metaDataFilepath);
+ SegmentStatusManager.readLoadMetadata(metaDataFilepath);
for (LoadMetadataDetails loadDetail : loadDetails) {
if (loadsToMerge.contains(loadDetail)) {
@@ -235,18 +231,18 @@ public final class CarbonDataMergerUtil {
}
}
- segmentUpdateStatusManager
- .writeLoadDetailsIntoFile(Arrays.asList(updateLists), timestamp);
- segmentStatusManager
- .writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(), loadDetails);
+ segmentUpdateStatusManager.writeLoadDetailsIntoFile(
+ Arrays.asList(updateLists), timestamp);
+ SegmentStatusManager.writeLoadDetailsIntoFile(
+ CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()), loadDetails);
status = true;
} else {
LOGGER.error("Not able to acquire the lock.");
status = false;
}
} catch (IOException e) {
- LOGGER.error("Error while updating metadata. The metadata file path is " + carbonTablePath
- .getMetadataDirectoryPath());
+ LOGGER.error("Error while updating metadata. The metadata file path is " +
+ CarbonTablePath.getMetadataPath(identifier.getTablePath()));
status = false;
} finally {
@@ -282,9 +278,9 @@ public final class CarbonDataMergerUtil {
String metaDataFilepath, String mergedLoadNumber, CarbonLoadModel carbonLoadModel,
CompactionType compactionType) throws IOException {
boolean tableStatusUpdationStatus = false;
- AbsoluteTableIdentifier absoluteTableIdentifier =
+ AbsoluteTableIdentifier identifier =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
- SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
+ SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
@@ -293,10 +289,7 @@ public final class CarbonDataMergerUtil {
LOGGER.info("Acquired lock for the table " + carbonLoadModel.getDatabaseName() + "."
+ carbonLoadModel.getTableName() + " for table status updation ");
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier);
-
- String statusFilePath = carbonTablePath.getTableStatusFilePath();
+ String statusFilePath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
LoadMetadataDetails[] loadDetails = SegmentStatusManager.readLoadMetadata(metaDataFilepath);
@@ -595,10 +588,6 @@ public final class CarbonDataMergerUtil {
List<LoadMetadataDetails> segmentsToBeMerged =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- CarbonTableIdentifier tableIdentifier =
- carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getCarbonTableIdentifier();
-
-
// total length
long totalLength = 0;
@@ -613,7 +602,7 @@ public final class CarbonDataMergerUtil {
String segId = segment.getLoadName();
// variable to store one segment size across partition.
long sizeOfOneSegmentAcrossPartition =
- getSizeOfSegment(tablePath, tableIdentifier, segId);
+ getSizeOfSegment(tablePath, segId);
// if size of a segment is greater than the Major compaction size. then ignore it.
if (sizeOfOneSegmentAcrossPartition > (compactionSize * 1024 * 1024)) {
@@ -652,35 +641,17 @@ public final class CarbonDataMergerUtil {
/**
* For calculating the size of the specified segment
* @param tablePath the store path of the segment
- * @param tableIdentifier identifier of table that the segment belong to
* @param segId segment id
* @return the data size of the segment
*/
- private static long getSizeOfSegment(String tablePath,
- CarbonTableIdentifier tableIdentifier, String segId) {
- String loadPath = getStoreLocation(tablePath, tableIdentifier, segId);
+ private static long getSizeOfSegment(String tablePath, String segId) {
+ String loadPath = CarbonTablePath.getSegmentPath(tablePath, segId);
CarbonFile segmentFolder =
FileFactory.getCarbonFile(loadPath, FileFactory.getFileType(loadPath));
return getSizeOfFactFileInLoad(segmentFolder);
}
/**
- * This method will get the store location for the given path, segemnt id and partition id
- *
- * @param tablePath
- * @param carbonTableIdentifier identifier of catbon table that the segment belong to
- * @param segmentId segment id
- * @return the store location of the segment
- */
- private static String getStoreLocation(String tablePath,
- CarbonTableIdentifier carbonTableIdentifier, String segmentId) {
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(tablePath, carbonTableIdentifier);
- return carbonTablePath.getCarbonDataDirectoryPath(segmentId);
- }
-
-
- /**
* Identify the segments to be merged based on the segment count
*
* @param listOfSegmentsAfterPreserve the list of segments after
@@ -1022,21 +993,19 @@ public final class CarbonDataMergerUtil {
* if UpdateDelta Files are more than IUD Compaction threshold.
*
* @param seg
- * @param absoluteTableIdentifier
+ * @param identifier
* @param segmentUpdateStatusManager
* @param numberDeltaFilesThreshold
* @return
*/
public static Boolean checkUpdateDeltaFilesInSeg(String seg,
- AbsoluteTableIdentifier absoluteTableIdentifier,
+ AbsoluteTableIdentifier identifier,
SegmentUpdateStatusManager segmentUpdateStatusManager, int numberDeltaFilesThreshold) {
CarbonFile[] updateDeltaFiles = null;
Set<String> uniqueBlocks = new HashSet<String>();
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
-
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(seg);
+ String segmentPath = CarbonTablePath.getSegmentPath(identifier.getTablePath(), seg);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
@@ -1282,15 +1251,12 @@ public final class CarbonDataMergerUtil {
CarbonUpdateUtil.updateSegmentStatus(segmentUpdateDetails, table, timestamp, true);
// Update the Table Status.
- String metaDataFilepath = table.getMetaDataFilepath();
- AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
-
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier);
+ String metaDataFilepath = table.getMetadataPath();
+ AbsoluteTableIdentifier identifier = table.getAbsoluteTableIdentifier();
- String tableStatusPath = carbonTablePath.getTableStatusFilePath();
+ String tableStatusPath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
- SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
+ SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
@@ -1304,7 +1270,7 @@ public final class CarbonDataMergerUtil {
+ " for table status updation");
LoadMetadataDetails[] listOfLoadFolderDetailsArray =
- segmentStatusManager.readLoadMetadata(metaDataFilepath);
+ SegmentStatusManager.readLoadMetadata(metaDataFilepath);
for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
if (loadMetadata.getLoadName().equalsIgnoreCase("0")) {
@@ -1313,7 +1279,7 @@ public final class CarbonDataMergerUtil {
}
}
try {
- segmentStatusManager
+ SegmentStatusManager
.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetailsArray);
} catch (IOException e) {
return false;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index ff65db2..8fc6e66 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -404,8 +404,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
.getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,
tempStoreLocation);
- setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonTable,
- carbonFactDataHandlerModel);
+ setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);
dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel,
CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
index 3d0700b..6f506b1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
@@ -72,8 +72,7 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
.getCarbonFactDataHandlerModel(loadModel, carbonTable, segProp, tableName,
tempStoreLocation);
- setDataFileAttributesInModel(loadModel, compactionType, carbonTable,
- carbonFactDataHandlerModel);
+ setDataFileAttributesInModel(loadModel, compactionType, carbonFactDataHandlerModel);
carbonFactDataHandlerModel.setCompactionFlow(true);
dataHandler = new CarbonFactDataHandlerColumnar(carbonFactDataHandlerModel);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 9f3c86f..bc87823 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -35,7 +35,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
import org.apache.carbondata.processing.datatypes.GenericDataType;
@@ -308,8 +307,7 @@ public class CarbonFactDataHandlerModel {
}
carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes);
String carbonDataDirectoryPath = CarbonDataProcessorUtil
- .checkAndCreateCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(),
- tableName, loadModel.getSegmentId());
+ .checkAndCreateCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getSegmentId());
carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
List<CarbonDimension> dimensionByTableName = carbonTable.getDimensionByTableName(tableName);
boolean[] isUseInvertedIndexes = new boolean[dimensionByTableName.size()];
@@ -334,10 +332,9 @@ public class CarbonFactDataHandlerModel {
* @return data directory path
*/
private static String getCarbonDataFolderLocation(CarbonDataLoadConfiguration configuration) {
- AbsoluteTableIdentifier absoluteTableIdentifier = configuration.getTableIdentifier();
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
+ AbsoluteTableIdentifier identifier = configuration.getTableIdentifier();
String carbonDataDirectoryPath =
- carbonTablePath.getCarbonDataDirectoryPath(configuration.getSegmentId());
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(), configuration.getSegmentId());
CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
return carbonDataDirectoryPath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index cfe6e31..ccde9e1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.ColumnType;
import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -44,7 +43,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.datatypes.ArrayDataType;
import org.apache.carbondata.processing.datatypes.GenericDataType;
@@ -143,12 +141,9 @@ public final class CarbonDataProcessorUtil {
String[] baseTmpStorePathArray = StringUtils.split(baseTempStorePath, File.pathSeparator);
String[] localDataFolderLocArray = new String[baseTmpStorePathArray.length];
- CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName);
for (int i = 0 ; i < baseTmpStorePathArray.length; i++) {
String tmpStore = baseTmpStorePathArray[i];
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(tmpStore, carbonTable.getCarbonTableIdentifier());
- String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
+ String carbonDataDirectoryPath = CarbonTablePath.getSegmentPath(tmpStore, segmentId);
localDataFolderLocArray[i] = carbonDataDirectoryPath + File.separator + taskId;
}
@@ -375,12 +370,9 @@ public final class CarbonDataProcessorUtil {
* @return data directory path
*/
public static String checkAndCreateCarbonStoreLocation(String factStoreLocation,
- String databaseName, String tableName, String segmentId) {
- CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName);
- CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(factStoreLocation, carbonTableIdentifier);
- String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
+ String segmentId) {
+ String carbonDataDirectoryPath = CarbonTablePath.getSegmentPath(
+ factStoreLocation, segmentId);
CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
return carbonDataDirectoryPath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 7be61d9..c2f4501 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -45,7 +45,6 @@ import org.apache.carbondata.core.fileoperations.FileWriteOperation;
import org.apache.carbondata.core.locks.CarbonLockUtil;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -54,7 +53,6 @@ import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.merger.NodeBlockRelation;
@@ -73,11 +71,8 @@ public final class CarbonLoaderUtil {
}
public static void deleteSegment(CarbonLoadModel loadModel, int currentLoad) {
- CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(loadModel.getTablePath(), carbonTable.getCarbonTableIdentifier());
-
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(currentLoad + "");
+ String segmentPath = CarbonTablePath.getSegmentPath(
+ loadModel.getTablePath(), currentLoad + "");
deleteStorePath(segmentPath);
}
@@ -90,33 +85,26 @@ public final class CarbonLoaderUtil {
*/
public static boolean isValidSegment(CarbonLoadModel loadModel,
int currentLoad) {
- CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema()
- .getCarbonTable();
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(
- loadModel.getTablePath(), carbonTable.getCarbonTableIdentifier());
int fileCount = 0;
- int partitionCount = carbonTable.getPartitionCount();
- for (int i = 0; i < partitionCount; i++) {
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(
- currentLoad + "");
- CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath,
- FileFactory.getFileType(segmentPath));
- CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return file.getName().endsWith(
- CarbonTablePath.getCarbonIndexExtension())
- || file.getName().endsWith(
- CarbonTablePath.getCarbonDataExtension());
- }
-
- });
- fileCount += files.length;
- if (files.length > 0) {
- return true;
+ String segmentPath = CarbonTablePath.getSegmentPath(
+ loadModel.getTablePath(), currentLoad + "");
+ CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath,
+ FileFactory.getFileType(segmentPath));
+ CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(
+ CarbonTablePath.getCarbonIndexExtension())
+ || file.getName().endsWith(
+ CarbonTablePath.getCarbonDataExtension());
}
+
+ });
+ fileCount += files.length;
+ if (files.length > 0) {
+ return true;
}
if (fileCount == 0) {
return false;
@@ -149,16 +137,15 @@ public final class CarbonLoaderUtil {
CarbonLoadModel loadModel, boolean loadStartEntry, boolean insertOverwrite)
throws IOException {
boolean status = false;
- AbsoluteTableIdentifier absoluteTableIdentifier =
+ AbsoluteTableIdentifier identifier =
loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- String metadataPath = carbonTablePath.getMetadataDirectoryPath();
+ String metadataPath = CarbonTablePath.getMetadataPath(identifier.getTablePath());
FileType fileType = FileFactory.getFileType(metadataPath);
if (!FileFactory.isFileExist(metadataPath, fileType)) {
FileFactory.mkdirs(metadataPath, fileType);
}
- String tableStatusPath = carbonTablePath.getTableStatusFilePath();
- SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
+ String tableStatusPath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
+ SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
int retryCount = CarbonLockUtil
.getLockProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK,
@@ -172,7 +159,8 @@ public final class CarbonLoaderUtil {
"Acquired lock for table" + loadModel.getDatabaseName() + "." + loadModel.getTableName()
+ " for table status updation");
LoadMetadataDetails[] listOfLoadFolderDetailsArray =
- SegmentStatusManager.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
+ SegmentStatusManager.readLoadMetadata(
+ CarbonTablePath.getMetadataPath(identifier.getTablePath()));
List<LoadMetadataDetails> listOfLoadFolderDetails =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<CarbonFile> staleFolders = new ArrayList<>();
@@ -198,13 +186,13 @@ public final class CarbonLoaderUtil {
// is triggered
for (LoadMetadataDetails entry : listOfLoadFolderDetails) {
if (entry.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS
- && segmentStatusManager.checkIfValidLoadInProgress(
- absoluteTableIdentifier, entry.getLoadName())) {
+ && SegmentStatusManager.checkIfValidLoadInProgress(
+ identifier, entry.getLoadName())) {
throw new RuntimeException("Already insert overwrite is in progress");
} else if (newMetaEntry.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS
&& entry.getSegmentStatus() == SegmentStatus.INSERT_IN_PROGRESS
- && segmentStatusManager.checkIfValidLoadInProgress(
- absoluteTableIdentifier, entry.getLoadName())) {
+ && SegmentStatusManager.checkIfValidLoadInProgress(
+ identifier, entry.getLoadName())) {
throw new RuntimeException("Already insert into or load is in progress");
}
}
@@ -227,7 +215,7 @@ public final class CarbonLoaderUtil {
entry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
// For insert overwrite, we will delete the old segment folder immediately
// So collect the old segments here
- addToStaleFolders(carbonTablePath, staleFolders, entry);
+ addToStaleFolders(identifier, staleFolders, entry);
}
}
}
@@ -236,7 +224,7 @@ public final class CarbonLoaderUtil {
// when no records are inserted then newSegmentEntry will be SegmentStatus.MARKED_FOR_DELETE
// so empty segment folder should be deleted
if (newMetaEntry.getSegmentStatus() == SegmentStatus.MARKED_FOR_DELETE) {
- addToStaleFolders(carbonTablePath, staleFolders, newMetaEntry);
+ addToStaleFolders(identifier, staleFolders, newMetaEntry);
}
SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetails
@@ -270,9 +258,10 @@ public final class CarbonLoaderUtil {
return status;
}
- private static void addToStaleFolders(CarbonTablePath carbonTablePath,
+ private static void addToStaleFolders(AbsoluteTableIdentifier identifier,
List<CarbonFile> staleFolders, LoadMetadataDetails entry) throws IOException {
- String path = carbonTablePath.getCarbonDataDirectoryPath(entry.getLoadName());
+ String path = CarbonTablePath.getSegmentPath(
+ identifier.getTablePath(), entry.getLoadName());
// add to the deletion list only if file exist else HDFS file system will throw
// exception while deleting the file if file path does not exist
if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
@@ -298,11 +287,9 @@ public final class CarbonLoaderUtil {
loadMetadataDetails.setLoadStartTime(loadStartTime);
}
- public static void writeLoadMetadata(AbsoluteTableIdentifier absoluteTableIdentifier,
+ public static void writeLoadMetadata(AbsoluteTableIdentifier identifier,
List<LoadMetadataDetails> listOfLoadFolderDetails) throws IOException {
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- String dataLoadLocation = carbonTablePath.getTableStatusFilePath();
+ String dataLoadLocation = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
DataOutputStream dataOutputStream;
Gson gsonObjectToWrite = new Gson();
@@ -838,10 +825,8 @@ public final class CarbonLoaderUtil {
* This method will get the store location for the given path, segment id and partition id
*/
public static void checkAndCreateCarbonDataLocation(String segmentId, CarbonTable carbonTable) {
- CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath(), carbonTableIdentifier);
- String segmentFolder = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
+ String segmentFolder = CarbonTablePath.getSegmentPath(
+ carbonTable.getTablePath(), segmentId);
CarbonUtil.checkAndCreateFolder(segmentFolder);
}
@@ -870,10 +855,8 @@ public final class CarbonLoaderUtil {
*/
public static Long addDataIndexSizeIntoMetaEntry(LoadMetadataDetails loadMetadataDetails,
String segmentId, CarbonTable carbonTable) throws IOException {
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath((carbonTable.getAbsoluteTableIdentifier()));
Map<String, Long> dataIndexSize =
- CarbonUtil.getDataSizeAndIndexSize(carbonTablePath, segmentId);
+ CarbonUtil.getDataSizeAndIndexSize(carbonTable.getTablePath(), segmentId);
Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
loadMetadataDetails.setDataSize(String.valueOf(dataSize));
Long indexSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
index f9f3e20..1fdce32 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
@@ -32,7 +32,6 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
public final class DeleteLoadFolders {
@@ -47,15 +46,14 @@ public final class DeleteLoadFolders {
/**
* returns segment path
*
- * @param absoluteTableIdentifier
+ * @param identifier
* @param oneLoad
* @return
*/
- private static String getSegmentPath(AbsoluteTableIdentifier absoluteTableIdentifier,
+ private static String getSegmentPath(AbsoluteTableIdentifier identifier,
LoadMetadataDetails oneLoad) {
- CarbonTablePath carbon = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
String segmentId = oneLoad.getLoadName();
- return carbon.getCarbonDataDirectoryPath(segmentId);
+ return CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
}
public static void physicalFactAndMeasureMetadataDeletion(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
index 7925b35..e059b35 100644
--- a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
+++ b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
@@ -73,12 +73,12 @@ public class BlockIndexStoreTest extends TestCase {
// file.length(), ColumnarFormatVersion.V1, null);
// CarbonTableIdentifier carbonTableIdentifier =
// new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "t3", "1");
-// AbsoluteTableIdentifier absoluteTableIdentifier =
+// AbsoluteTableIdentifier identifier =
// new AbsoluteTableIdentifier("/src/test/resources", carbonTableIdentifier);
// try {
//
// List<TableBlockUniqueIdentifier> tableBlockInfoList =
-// getTableBlockUniqueIdentifierList(Arrays.asList(new TableBlockInfo[] { info }), absoluteTableIdentifier);
+// getTableBlockUniqueIdentifierList(Arrays.asList(new TableBlockInfo[] { info }), identifier);
// List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockInfoList);
// assertTrue(loadAndGetBlocks.size() == 1);
// } catch (Exception e) {
@@ -86,7 +86,7 @@ public class BlockIndexStoreTest extends TestCase {
// }
// List<String> segmentIds = new ArrayList<>();
// segmentIds.add(info.getSegmentId());
-// cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
+// cache.removeTableBlocks(segmentIds, identifier);
// }
//
private List<TableBlockUniqueIdentifier> getTableBlockUniqueIdentifierList(List<TableBlockInfo> tableBlockInfos,
@@ -122,19 +122,19 @@ public class BlockIndexStoreTest extends TestCase {
//
// CarbonTableIdentifier carbonTableIdentifier =
// new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "t3", "1");
-// AbsoluteTableIdentifier absoluteTableIdentifier =
+// AbsoluteTableIdentifier identifier =
// new AbsoluteTableIdentifier("/src/test/resources", carbonTableIdentifier);
// ExecutorService executor = Executors.newFixedThreadPool(3);
// executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info, info1 }),
-// absoluteTableIdentifier));
+// identifier));
// executor.submit(
// new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info2, info3, info4 }),
-// absoluteTableIdentifier));
+// identifier));
// executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info, info1 }),
-// absoluteTableIdentifier));
+// identifier));
// executor.submit(
// new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info2, info3, info4 }),
-// absoluteTableIdentifier));
+// identifier));
// executor.shutdown();
// try {
// executor.awaitTermination(1, TimeUnit.DAYS);
@@ -145,7 +145,7 @@ public class BlockIndexStoreTest extends TestCase {
// Arrays.asList(new TableBlockInfo[] { info, info1, info2, info3, info4 });
// try {
// List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
-// getTableBlockUniqueIdentifierList(tableBlockInfos, absoluteTableIdentifier);
+// getTableBlockUniqueIdentifierList(tableBlockInfos, identifier);
// List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockUniqueIdentifiers);
// assertTrue(loadAndGetBlocks.size() == 5);
// } catch (Exception e) {
@@ -155,7 +155,7 @@ public class BlockIndexStoreTest extends TestCase {
// for (TableBlockInfo tableBlockInfo : tableBlockInfos) {
// segmentIds.add(tableBlockInfo.getSegmentId());
// }
-// cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
+// cache.removeTableBlocks(segmentIds, identifier);
// }
//
// public void testloadAndGetTaskIdToSegmentsMapForDifferentSegmentLoadedConcurrently()
@@ -193,18 +193,18 @@ public class BlockIndexStoreTest extends TestCase {
//
// CarbonTableIdentifier carbonTableIdentifier =
// new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "t3", "1");
-// AbsoluteTableIdentifier absoluteTableIdentifier =
+// AbsoluteTableIdentifier identifier =
// new AbsoluteTableIdentifier("/src/test/resources", carbonTableIdentifier);
// ExecutorService executor = Executors.newFixedThreadPool(3);
// executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info, info1 }),
-// absoluteTableIdentifier));
+// identifier));
// executor.submit(
// new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info2, info3, info4 }),
-// absoluteTableIdentifier));
+// identifier));
// executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info5, info6 }),
-// absoluteTableIdentifier));
+// identifier));
// executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info7 }),
-// absoluteTableIdentifier));
+// identifier));
//
// executor.shutdown();
// try {
@@ -217,7 +217,7 @@ public class BlockIndexStoreTest extends TestCase {
// .asList(new TableBlockInfo[] { info, info1, info2, info3, info4, info5, info6, info7 });
// try {
// List<TableBlockUniqueIdentifier> blockUniqueIdentifierList =
-// getTableBlockUniqueIdentifierList(tableBlockInfos, absoluteTableIdentifier);
+// getTableBlockUniqueIdentifierList(tableBlockInfos, identifier);
// List<AbstractIndex> loadAndGetBlocks = cache.getAll(blockUniqueIdentifierList);
// assertTrue(loadAndGetBlocks.size() == 8);
// } catch (Exception e) {
@@ -227,7 +227,7 @@ public class BlockIndexStoreTest extends TestCase {
// for (TableBlockInfo tableBlockInfo : tableBlockInfos) {
// segmentIds.add(tableBlockInfo.getSegmentId());
// }
-// cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
+// cache.removeTableBlocks(segmentIds, identifier);
// }
private class BlockLoaderThread implements Callable<Void> {
@@ -250,7 +250,7 @@ public class BlockIndexStoreTest extends TestCase {
}
private static File getPartFile() {
- String path = StoreCreator.getAbsoluteTableIdentifier().getTablePath()
+ String path = StoreCreator.getIdentifier().getTablePath()
+ "/Fact/Part0/Segment_0";
File file = new File(path);
File[] files = file.listFiles();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
index 7f0aef6..d42dcde 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
@@ -64,7 +64,6 @@ import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
@@ -98,14 +97,14 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
*/
public class StoreCreator {
- private static AbsoluteTableIdentifier absoluteTableIdentifier;
+ private static AbsoluteTableIdentifier identifier;
private static String storePath = "";
static {
try {
storePath = new File("target/store").getCanonicalPath();
String dbName = "testdb";
String tableName = "testtable";
- absoluteTableIdentifier =
+ identifier =
AbsoluteTableIdentifier.from(
storePath + "/testdb/testtable",
new CarbonTableIdentifier(dbName, tableName, UUID.randomUUID().toString()));
@@ -114,8 +113,8 @@ public class StoreCreator {
}
}
- public static AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
- return absoluteTableIdentifier;
+ public static AbsoluteTableIdentifier getIdentifier() {
+ return identifier;
}
/**
@@ -134,12 +133,12 @@ public class StoreCreator {
CarbonDataLoadSchema schema = new CarbonDataLoadSchema(table);
CarbonLoadModel loadModel = new CarbonLoadModel();
loadModel.setCarbonDataLoadSchema(schema);
- loadModel.setDatabaseName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName());
- loadModel.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
- loadModel.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
+ loadModel.setDatabaseName(identifier.getCarbonTableIdentifier().getDatabaseName());
+ loadModel.setTableName(identifier.getCarbonTableIdentifier().getTableName());
+ loadModel.setTableName(identifier.getCarbonTableIdentifier().getTableName());
loadModel.setFactFilePath(factFilePath);
loadModel.setLoadMetadataDetails(new ArrayList<LoadMetadataDetails>());
- loadModel.setTablePath(absoluteTableIdentifier.getTablePath());
+ loadModel.setTablePath(identifier.getTablePath());
loadModel.setDateFormat(null);
loadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
@@ -175,9 +174,9 @@ public class StoreCreator {
private static CarbonTable createTable() throws IOException {
TableInfo tableInfo = new TableInfo();
- tableInfo.setDatabaseName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName());
+ tableInfo.setDatabaseName(identifier.getCarbonTableIdentifier().getDatabaseName());
TableSchema tableSchema = new TableSchema();
- tableSchema.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
+ tableSchema.setTableName(identifier.getCarbonTableIdentifier().getTableName());
List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
ArrayList<Encoding> encodings = new ArrayList<>();
encodings.add(Encoding.DICTIONARY);
@@ -257,16 +256,13 @@ public class StoreCreator {
tableSchema.setSchemaEvalution(schemaEvol);
tableSchema.setTableId(UUID.randomUUID().toString());
tableInfo.setTableUniqueName(
- absoluteTableIdentifier.getCarbonTableIdentifier().getTableUniqueName()
+ identifier.getCarbonTableIdentifier().getTableUniqueName()
);
tableInfo.setLastUpdatedTime(System.currentTimeMillis());
tableInfo.setFactTable(tableSchema);
- tableInfo.setTablePath(absoluteTableIdentifier.getTablePath());
+ tableInfo.setTablePath(identifier.getTablePath());
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
- String schemaFilePath = carbonTablePath.getSchemaFilePath();
+ String schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath());
String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath);
CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
@@ -329,7 +325,7 @@ public class StoreCreator {
writer.close();
writer.commit();
Dictionary dict = (Dictionary) dictCache.get(
- new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier,
+ new DictionaryColumnUniqueIdentifier(identifier,
columnIdentifier, dims.get(i).getDataType()));
CarbonDictionarySortInfoPreparator preparator =
new CarbonDictionarySortInfoPreparator();
@@ -444,7 +440,7 @@ public class StoreCreator {
loadMetadataDetails.setLoadStartTime(loadMetadataDetails.getTimeStamp(readCurrentTime()));
listOfLoadFolderDetails.add(loadMetadataDetails);
- String dataLoadLocation = schema.getCarbonTable().getMetaDataFilepath() + File.separator
+ String dataLoadLocation = schema.getCarbonTable().getMetadataPath() + File.separator
+ CarbonCommonConstants.LOADMETADATA_FILENAME;
DataOutputStream dataOutputStream;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
index 7b823ac..8c9889d 100644
--- a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
+++ b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
@@ -38,7 +38,6 @@ import org.apache.carbondata.core.statusmanager.FileFormat;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonIndexFileWriter;
import org.apache.carbondata.format.BlockIndex;
@@ -60,8 +59,6 @@ public class StreamSegment {
* get stream segment or create new stream segment if not exists
*/
public static String open(CarbonTable table) throws IOException {
- CarbonTablePath tablePath =
- CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier());
SegmentStatusManager segmentStatusManager =
new SegmentStatusManager(table.getAbsoluteTableIdentifier());
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
@@ -72,7 +69,8 @@ public class StreamSegment {
+ " for stream table get or create segment");
LoadMetadataDetails[] details =
- SegmentStatusManager.readLoadMetadata(tablePath.getMetadataDirectoryPath());
+ SegmentStatusManager.readLoadMetadata(
+ CarbonTablePath.getMetadataPath(table.getTablePath()));
LoadMetadataDetails streamSegment = null;
for (LoadMetadataDetails detail : details) {
if (FileFormat.ROW_V1 == detail.getFileFormat()) {
@@ -97,8 +95,8 @@ public class StreamSegment {
newDetails[i] = details[i];
}
newDetails[i] = newDetail;
- SegmentStatusManager
- .writeLoadDetailsIntoFile(tablePath.getTableStatusFilePath(), newDetails);
+ SegmentStatusManager.writeLoadDetailsIntoFile(
+ CarbonTablePath.getTableStatusFilePath(table.getTablePath()), newDetails);
return newDetail.getLoadName();
} else {
return streamSegment.getLoadName();
@@ -126,8 +124,6 @@ public class StreamSegment {
*/
public static String close(CarbonTable table, String segmentId)
throws IOException {
- CarbonTablePath tablePath =
- CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier());
SegmentStatusManager segmentStatusManager =
new SegmentStatusManager(table.getAbsoluteTableIdentifier());
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
@@ -138,7 +134,8 @@ public class StreamSegment {
+ " for stream table finish segment");
LoadMetadataDetails[] details =
- SegmentStatusManager.readLoadMetadata(tablePath.getMetadataDirectoryPath());
+ SegmentStatusManager.readLoadMetadata(
+ CarbonTablePath.getMetadataPath(table.getTablePath()));
for (LoadMetadataDetails detail : details) {
if (segmentId.equals(detail.getLoadName())) {
detail.setLoadEndTime(System.currentTimeMillis());
@@ -162,7 +159,8 @@ public class StreamSegment {
}
newDetails[i] = newDetail;
SegmentStatusManager
- .writeLoadDetailsIntoFile(tablePath.getTableStatusFilePath(), newDetails);
+ .writeLoadDetailsIntoFile(CarbonTablePath.getTableStatusFilePath(
+ table.getTablePath()), newDetails);
return newDetail.getLoadName();
} else {
LOGGER.error(
@@ -192,7 +190,7 @@ public class StreamSegment {
try {
if (statusLock.lockWithRetries()) {
LoadMetadataDetails[] details =
- SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath());
+ SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath());
boolean updated = false;
for (LoadMetadataDetails detail : details) {
if (SegmentStatus.STREAMING == detail.getSegmentStatus()) {
@@ -202,10 +200,8 @@ public class StreamSegment {
}
}
if (updated) {
- CarbonTablePath tablePath =
- CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier());
SegmentStatusManager.writeLoadDetailsIntoFile(
- tablePath.getTableStatusFilePath(),
+ CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath()),
details);
}
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 197cb14..186d100 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -37,7 +37,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.CarbonUtil
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonProjection}
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
import org.apache.carbondata.hadoop.streaming.{CarbonStreamInputFormat, CarbonStreamRecordReader}
@@ -216,7 +216,6 @@ object StreamHandoffRDD {
): Unit = {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val identifier = carbonTable.getAbsoluteTableIdentifier
- val tablePath = CarbonStorePath.getCarbonTablePath(identifier)
var continueHandoff = false
// require handoff lock on table
val lock = CarbonLockFactory.getCarbonLockObj(identifier, LockUsage.HANDOFF_LOCK)
@@ -233,7 +232,7 @@ object StreamHandoffRDD {
try {
if (statusLock.lockWithRetries()) {
loadMetadataDetails = SegmentStatusManager.readLoadMetadata(
- tablePath.getMetadataDirectoryPath)
+ CarbonTablePath.getMetadataPath(identifier.getTablePath))
}
} finally {
if (null != statusLock) {
@@ -355,19 +354,16 @@ object StreamHandoffRDD {
loadModel: CarbonLoadModel
): Boolean = {
var status = false
- val metaDataFilepath =
- loadModel.getCarbonDataLoadSchema().getCarbonTable().getMetaDataFilepath()
- val identifier =
- loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier()
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier)
- val metadataPath = carbonTablePath.getMetadataDirectoryPath()
+ val metaDataFilepath = loadModel.getCarbonDataLoadSchema.getCarbonTable.getMetadataPath
+ val identifier = loadModel.getCarbonDataLoadSchema.getCarbonTable.getAbsoluteTableIdentifier
+ val metadataPath = CarbonTablePath.getMetadataPath(identifier.getTablePath)
val fileType = FileFactory.getFileType(metadataPath)
if (!FileFactory.isFileExist(metadataPath, fileType)) {
FileFactory.mkdirs(metadataPath, fileType)
}
- val tableStatusPath = carbonTablePath.getTableStatusFilePath()
+ val tableStatusPath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath)
val segmentStatusManager = new SegmentStatusManager(identifier)
- val carbonLock = segmentStatusManager.getTableStatusLock()
+ val carbonLock = segmentStatusManager.getTableStatusLock
try {
if (carbonLock.lockWithRetries()) {
LOGGER.info(
@@ -400,7 +396,7 @@ object StreamHandoffRDD {
status = true
} else {
LOGGER.error("Not able to acquire the lock for Table status updation for table " + loadModel
- .getDatabaseName() + "." + loadModel.getTableName());
+ .getDatabaseName() + "." + loadModel.getTableName())
}
} finally {
if (carbonLock.unlock()) {
@@ -411,6 +407,6 @@ object StreamHandoffRDD {
"." + loadModel.getTableName() + " during table status updation")
}
}
- return status
+ status
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index f2274be..c417fbe 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -31,7 +31,7 @@ import org.apache.carbondata.core.dictionary.service.NonSecureDictionaryServiceP
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
@@ -126,16 +126,14 @@ object StreamSinkFactory {
* @return
*/
private def getStreamSegmentId(carbonTable: CarbonTable): String = {
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val fileType = FileFactory.getFileType(carbonTablePath.getMetadataDirectoryPath)
- if (!FileFactory.isFileExist(carbonTablePath.getMetadataDirectoryPath, fileType)) {
+ val segmentId = StreamSegment.open(carbonTable)
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentId)
+ val fileType = FileFactory.getFileType(segmentDir)
+ if (!FileFactory.isFileExist(segmentDir, fileType)) {
// Create table directory path, in case of enabling hive metastore first load may not have
// table folder created.
- FileFactory.mkdirs(carbonTablePath.getMetadataDirectoryPath, fileType)
+ FileFactory.mkdirs(segmentDir, fileType)
}
- val segmentId = StreamSegment.open(carbonTable)
- val segmentDir = carbonTablePath.getSegmentDir(segmentId)
if (FileFactory.isFileExist(segmentDir, fileType)) {
// recover fault
StreamSegment.recoverSegmentIfRequired(segmentDir)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index 45bc19a..ff483e5 100644
--- a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++ b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -40,7 +40,7 @@ import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.stats.QueryStatistic
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.hadoop.streaming.CarbonStreamOutputFormat
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
@@ -62,9 +62,7 @@ class CarbonAppendableStreamSink(
carbonLoadModel: CarbonLoadModel,
server: Option[DictionaryServer]) extends Sink {
- private val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- private val fileLogPath = carbonTablePath.getStreamingLogDir
+ private val fileLogPath = CarbonTablePath.getStreamingLogDir(carbonTable.getTablePath)
private val fileLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, sparkSession, fileLogPath)
// prepare configuration
private val hadoopConf = {
@@ -149,12 +147,12 @@ class CarbonAppendableStreamSink(
* if the directory size of current segment beyond the threshold, hand off new segment
*/
private def checkOrHandOffSegment(): Unit = {
- val segmentDir = carbonTablePath.getSegmentDir(currentSegmentId)
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, currentSegmentId)
val fileType = FileFactory.getFileType(segmentDir)
if (segmentMaxSize <= StreamSegment.size(segmentDir)) {
val newSegmentId = StreamSegment.close(carbonTable, currentSegmentId)
currentSegmentId = newSegmentId
- val newSegmentDir = carbonTablePath.getSegmentDir(currentSegmentId)
+ val newSegmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, currentSegmentId)
FileFactory.mkdirs(newSegmentDir, fileType)
// TODO trigger hand off operation
@@ -250,15 +248,13 @@ object CarbonAppendableStreamSink {
}
// update data file info in index file
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- StreamSegment.updateIndexFile(tablePath.getSegmentDir(segmentId))
+ StreamSegment.updateIndexFile(
+ CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentId))
} catch {
// catch fault of executor side
case t: Throwable =>
- val tablePath =
- CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = tablePath.getSegmentDir(segmentId)
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentId)
StreamSegment.recoverSegmentIfRequired(segmentDir)
LOGGER.error(t, s"Aborting job ${ job.getJobID }.")
committer.abortJob(job)
[48/50] [abbrv] carbondata git commit: [CARBONDATA-1480]Min Max Index
Example for DataMap
Posted by ra...@apache.org.
[CARBONDATA-1480]Min Max Index Example for DataMap
Datamap Example. Implementation of Min Max Index through Datamap. And Using the Index while prunning.
This closes #1359
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/cae74a8c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/cae74a8c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/cae74a8c
Branch: refs/heads/fgdatamap
Commit: cae74a8cecea74e8899a87dcb7d12e0dec1b8069
Parents: e502c59
Author: sounakr <so...@gmail.com>
Authored: Thu Sep 28 16:21:05 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Feb 5 12:34:25 2018 +0530
----------------------------------------------------------------------
.../core/datamap/DataMapStoreManager.java | 16 +-
.../carbondata/core/datamap/TableDataMap.java | 18 +-
.../carbondata/core/datamap/dev/DataMap.java | 14 +-
.../core/datamap/dev/DataMapWriter.java | 3 +-
.../indexstore/SegmentPropertiesFetcher.java | 36 +++
.../blockletindex/BlockletDataMap.java | 9 +-
.../blockletindex/BlockletDataMapFactory.java | 33 ++-
datamap/examples/pom.xml | 111 ++++++++++
.../datamap/examples/BlockletMinMax.java | 41 ++++
.../datamap/examples/MinMaxDataMap.java | 143 ++++++++++++
.../datamap/examples/MinMaxDataMapFactory.java | 114 ++++++++++
.../datamap/examples/MinMaxDataWriter.java | 221 +++++++++++++++++++
.../examples/MinMaxIndexBlockDetails.java | 77 +++++++
.../MinMaxDataMapExample.scala | 77 +++++++
.../testsuite/datamap/DataMapWriterSuite.scala | 2 +-
pom.xml | 2 +
.../datamap/DataMapWriterListener.java | 4 +-
.../store/writer/AbstractFactDataWriter.java | 7 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 3 +
19 files changed, 900 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index d30483a..90e5fff 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -103,7 +104,7 @@ public final class DataMapStoreManager {
tableDataMaps = new ArrayList<>();
}
TableDataMap dataMap = getTableDataMap(dataMapName, tableDataMaps);
- if (dataMap != null) {
+ if (dataMap != null && dataMap.getDataMapName().equalsIgnoreCase(dataMapName)) {
throw new RuntimeException("Already datamap exists in that path with type " + dataMapName);
}
@@ -113,12 +114,15 @@ public final class DataMapStoreManager {
DataMapFactory dataMapFactory = factoryClass.newInstance();
dataMapFactory.init(identifier, dataMapName);
BlockletDetailsFetcher blockletDetailsFetcher;
+ SegmentPropertiesFetcher segmentPropertiesFetcher = null;
if (dataMapFactory instanceof BlockletDetailsFetcher) {
blockletDetailsFetcher = (BlockletDetailsFetcher) dataMapFactory;
} else {
blockletDetailsFetcher = getBlockletDetailsFetcher(identifier);
}
- dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory, blockletDetailsFetcher);
+ segmentPropertiesFetcher = (SegmentPropertiesFetcher) blockletDetailsFetcher;
+ dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory, blockletDetailsFetcher,
+ segmentPropertiesFetcher);
} catch (Exception e) {
LOGGER.error(e);
throw new RuntimeException(e);
@@ -128,11 +132,11 @@ public final class DataMapStoreManager {
return dataMap;
}
- private TableDataMap getTableDataMap(String dataMapName,
- List<TableDataMap> tableDataMaps) {
+ private TableDataMap getTableDataMap(String dataMapName, List<TableDataMap> tableDataMaps) {
TableDataMap dataMap = null;
- for (TableDataMap tableDataMap: tableDataMaps) {
- if (tableDataMap.getDataMapName().equals(dataMapName)) {
+ for (TableDataMap tableDataMap : tableDataMaps) {
+ if (tableDataMap.getDataMapName().equals(dataMapName) || (!tableDataMap.getDataMapName()
+ .equals(""))) {
dataMap = tableDataMap;
break;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 9c84891..1c80703 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -23,9 +23,11 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.events.Event;
@@ -46,15 +48,19 @@ public final class TableDataMap extends OperationEventListener {
private BlockletDetailsFetcher blockletDetailsFetcher;
+ private SegmentPropertiesFetcher segmentPropertiesFetcher;
+
/**
* It is called to initialize and load the required table datamap metadata.
*/
public TableDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
- DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher) {
+ DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher,
+ SegmentPropertiesFetcher segmentPropertiesFetcher) {
this.identifier = identifier;
this.dataMapName = dataMapName;
this.dataMapFactory = dataMapFactory;
this.blockletDetailsFetcher = blockletDetailsFetcher;
+ this.segmentPropertiesFetcher = segmentPropertiesFetcher;
}
/**
@@ -67,11 +73,13 @@ public final class TableDataMap extends OperationEventListener {
public List<ExtendedBlocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp,
List<String> partitions) throws IOException {
List<ExtendedBlocklet> blocklets = new ArrayList<>();
+ SegmentProperties segmentProperties;
for (String segmentId : segmentIds) {
List<Blocklet> pruneBlocklets = new ArrayList<>();
List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
+ segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId);
for (DataMap dataMap : dataMaps) {
- pruneBlocklets.addAll(dataMap.prune(filterExp, partitions));
+ pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions));
}
blocklets.addAll(addSegmentId(blockletDetailsFetcher
.getExtendedBlocklets(pruneBlocklets, segmentId), segmentId));
@@ -123,7 +131,11 @@ public final class TableDataMap extends OperationEventListener {
List<Blocklet> blocklets = new ArrayList<>();
List<DataMap> dataMaps = dataMapFactory.getDataMaps(distributable);
for (DataMap dataMap : dataMaps) {
- blocklets.addAll(dataMap.prune(filterExp, partitions));
+ blocklets.addAll(
+ dataMap.prune(
+ filterExp,
+ segmentPropertiesFetcher.getSegmentProperties(distributable.getSegmentId()),
+ partitions));
}
for (Blocklet blocklet: blocklets) {
ExtendedBlocklet detailedBlocklet =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index 16be1ac..dfe97e3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datamap.dev;
import java.io.IOException;
import java.util.List;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -34,23 +35,14 @@ public interface DataMap {
void init(DataMapModel dataMapModel) throws MemoryException, IOException;
/**
- * Prune the datamap with filter expression. It returns the list of
- * blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
- */
- List<Blocklet> prune(FilterResolverIntf filterExp);
-
- // TODO Move this method to Abstract class
- /**
* Prune the datamap with filter expression and partition information. It returns the list of
* blocklets where these filters can exist.
*
* @param filterExp
* @return
*/
- List<Blocklet> prune(FilterResolverIntf filterExp, List<String> partitions);
+ List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<String> partitions);
// TODO Move this method to Abstract class
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
index 28163d7..413eaa5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
@@ -27,7 +27,7 @@ public interface DataMapWriter {
* Start of new block notification.
* @param blockId file name of the carbondata file
*/
- void onBlockStart(String blockId);
+ void onBlockStart(String blockId, String blockPath);
/**
* End of block notification
@@ -45,7 +45,6 @@ public interface DataMapWriter {
* @param blockletId sequence number of blocklet in the block
*/
void onBlockletEnd(int blockletId);
-
/**
* Add the column pages row to the datamap, order of pages is same as `indexColumns` in
* DataMapMeta returned in DataMapFactory.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java b/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java
new file mode 100644
index 0000000..ec2ae93
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/SegmentPropertiesFetcher.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.indexstore;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+
+/**
+ * Fetches the detailed segmentProperties which has more information to execute the query
+ */
+public interface SegmentPropertiesFetcher {
+
+ /**
+ * get the Segment properties based on the SegmentID.
+ * @param segmentId
+ * @return
+ * @throws IOException
+ */
+ SegmentProperties getSegmentProperties(String segmentId) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index b097c66..d331c2b 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -619,8 +619,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
return false;
}
- @Override
- public List<Blocklet> prune(FilterResolverIntf filterExp) {
+ private List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties) {
if (unsafeMemoryDMStore.getRowCount() == 0) {
return new ArrayList<>();
}
@@ -689,7 +688,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
return blocklets;
}
- @Override public List<Blocklet> prune(FilterResolverIntf filterExp, List<String> partitions) {
+ @Override
+ public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<String> partitions) {
if (unsafeMemoryDMStore.getRowCount() == 0) {
return new ArrayList<>();
}
@@ -711,7 +712,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
}
// Prune with filters if the partitions are existed in this datamap
- return prune(filterExp);
+ return prune(filterExp, segmentProperties);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 2e2cab5..61e5ceb 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -30,13 +30,18 @@ import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -48,13 +53,17 @@ import org.apache.hadoop.fs.RemoteIterator;
/**
* Table map for blocklet
*/
-public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFetcher {
+public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFetcher,
+ SegmentPropertiesFetcher {
private AbsoluteTableIdentifier identifier;
// segmentId -> list of index file
private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
+ // segmentId -> SegmentProperties.
+ private Map<String, SegmentProperties> segmentPropertiesMap = new HashMap<>();
+
private Cache<TableBlockIndexUniqueIdentifier, DataMap> cache;
@Override
@@ -170,6 +179,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
@Override
public void clear(String segmentId) {
+ segmentPropertiesMap.remove(segmentId);
List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segmentId);
if (blockIndexes != null) {
for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
@@ -221,4 +231,25 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
// TODO: pass SORT_COLUMNS into this class
return null;
}
+
+ @Override public SegmentProperties getSegmentProperties(String segmentId) throws IOException {
+ SegmentProperties segmentProperties = segmentPropertiesMap.get(segmentId);
+ if (segmentProperties == null) {
+ int[] columnCardinality;
+ List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
+ getTableBlockIndexUniqueIdentifiers(segmentId);
+ DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
+ List<DataFileFooter> indexInfo =
+ fileFooterConverter.getIndexInfo(tableBlockIndexUniqueIdentifiers.get(0).getFilePath());
+ for (DataFileFooter fileFooter : indexInfo) {
+ List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
+ if (segmentProperties == null) {
+ columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
+ segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
+ }
+ }
+ segmentPropertiesMap.put(segmentId, segmentProperties);
+ }
+ return segmentProperties;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/datamap/examples/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/examples/pom.xml b/datamap/examples/pom.xml
new file mode 100644
index 0000000..6832e62
--- /dev/null
+++ b/datamap/examples/pom.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-parent</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>carbondata-datamap-examples</artifactId>
+ <name>Apache CarbonData :: Datamap Examples</name>
+
+ <properties>
+ <dev.path>${basedir}/../../dev</dev.path>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-spark2</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive-thriftserver_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-repl_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.10</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-repl_${scala.binary.version}</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/minmaxdatamap/main/java</sourceDirectory>
+ <resources>
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>CARBON_EXAMPLESLogResource.properties</include>
+ </includes>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <version>2.15.2</version>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ <phase>compile</phase>
+ </execution>
+ <execution>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java
new file mode 100644
index 0000000..e6968fe
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/BlockletMinMax.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+
+public class BlockletMinMax {
+ private byte[][] Min;
+
+ private byte[][] Max;
+
+ public byte[][] getMin() {
+ return Min;
+ }
+
+ public void setMin(byte[][] min) {
+ Min = min;
+ }
+
+ public byte[][] getMax() {
+ return Max;
+ }
+
+ public void setMax(byte[][] max) {
+ Max = max;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
new file mode 100644
index 0000000..2ad6327
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+/**
+ * Datamap implementation for min max blocklet.
+ */
+public class MinMaxDataMap implements DataMap {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(MinMaxDataMap.class.getName());
+
+ public static final String NAME = "clustered.minmax.btree.blocklet";
+
+ private String filePath;
+
+ private MinMaxIndexBlockDetails[] readMinMaxDataMap;
+
+ @Override public void init(String filePath) throws MemoryException, IOException {
+ this.filePath = filePath;
+ CarbonFile[] listFiles = getCarbonMinMaxIndexFiles(filePath, "0");
+ for (int i = 0; i < listFiles.length; i++) {
+ readMinMaxDataMap = readJson(listFiles[i].getPath());
+ }
+ }
+
+ private CarbonFile[] getCarbonMinMaxIndexFiles(String filePath, String segmentId) {
+ String path = filePath.substring(0, filePath.lastIndexOf("/") + 1);
+ CarbonFile carbonFile = FileFactory.getCarbonFile(path);
+ return carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(".minmaxindex");
+ }
+ });
+ }
+
+ public MinMaxIndexBlockDetails[] readJson(String filePath) throws IOException {
+ Gson gsonObjectToRead = new Gson();
+ DataInputStream dataInputStream = null;
+ BufferedReader buffReader = null;
+ InputStreamReader inStream = null;
+ MinMaxIndexBlockDetails[] readMinMax = null;
+ AtomicFileOperations fileOperation =
+ new AtomicFileOperationsImpl(filePath, FileFactory.getFileType(filePath));
+
+ try {
+ if (!FileFactory.isFileExist(filePath, FileFactory.getFileType(filePath))) {
+ return null;
+ }
+ dataInputStream = fileOperation.openForRead();
+ inStream = new InputStreamReader(dataInputStream,
+ CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
+ buffReader = new BufferedReader(inStream);
+ readMinMax = gsonObjectToRead.fromJson(buffReader, MinMaxIndexBlockDetails[].class);
+ } catch (IOException e) {
+ return null;
+ } finally {
+ CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
+ }
+ return readMinMax;
+ }
+
+ /**
+ * Block Prunning logic for Min Max DataMap.
+ *
+ * @param filterExp
+ * @param segmentProperties
+ * @return
+ */
+ @Override public List<Blocklet> prune(FilterResolverIntf filterExp,
+ SegmentProperties segmentProperties) {
+ List<Blocklet> blocklets = new ArrayList<>();
+
+ if (filterExp == null) {
+ for (int i = 0; i < readMinMaxDataMap.length; i++) {
+ blocklets.add(new Blocklet(readMinMaxDataMap[i].getFilePath(),
+ String.valueOf(readMinMaxDataMap[i].getBlockletId())));
+ }
+ } else {
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
+ int startIndex = 0;
+ while (startIndex < readMinMaxDataMap.length) {
+ BitSet bitSet = filterExecuter.isScanRequired(readMinMaxDataMap[startIndex].getMaxValues(),
+ readMinMaxDataMap[startIndex].getMinValues());
+ if (!bitSet.isEmpty()) {
+ blocklets.add(new Blocklet(readMinMaxDataMap[startIndex].getFilePath(),
+ String.valueOf(readMinMaxDataMap[startIndex].getBlockletId())));
+ }
+ startIndex++;
+ }
+ }
+ return blocklets;
+ }
+
+ @Override
+ public void clear() {
+ readMinMaxDataMap = null;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
new file mode 100644
index 0000000..b196d0d
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.events.ChangeEvent;
+import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+
+
+/**
+ * Min Max DataMap Factory
+ */
+public class MinMaxDataMapFactory implements DataMapFactory {
+
+ private AbsoluteTableIdentifier identifier;
+
+ @Override
+ public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ this.identifier = identifier;
+ }
+
+ /**
+ * createWriter will return the MinMaxDataWriter.
+ * @param segmentId
+ * @return
+ */
+ @Override
+ public DataMapWriter createWriter(String segmentId) {
+ return new MinMaxDataWriter();
+ }
+
+ /**
+ * getDataMaps Factory method Initializes the Min Max Data Map and returns.
+ * @param segmentId
+ * @return
+ * @throws IOException
+ */
+ @Override public List<DataMap> getDataMaps(String segmentId) throws IOException {
+ List<DataMap> dataMapList = new ArrayList<>();
+ // Form a dataMap of Type MinMaxDataMap.
+ MinMaxDataMap dataMap = new MinMaxDataMap();
+ try {
+ dataMap.init(identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + File.separator);
+ } catch (MemoryException ex) {
+
+ }
+ dataMapList.add(dataMap);
+ return dataMapList;
+ }
+
+ /**
+ *
+ * @param segmentId
+ * @return
+ */
+ @Override public List<DataMapDistributable> toDistributable(String segmentId) {
+ return null;
+ }
+
+ /**
+ * Clear the DataMap.
+ * @param segmentId
+ */
+ @Override public void clear(String segmentId) {
+ }
+
+ /**
+ * Clearing the data map.
+ */
+ @Override
+ public void clear() {
+ }
+
+ @Override public DataMap getDataMap(DataMapDistributable distributable) {
+ return null;
+ }
+
+ @Override
+ public void fireEvent(ChangeEvent event) {
+
+ }
+
+ @Override
+ public DataMapMeta getMeta() {
+ return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")), FilterType.EQUALTO);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
new file mode 100644
index 0000000..78544d3
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+public class MinMaxDataWriter implements DataMapWriter {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(TableInfo.class.getName());
+
+ private byte[][] pageLevelMin, pageLevelMax;
+
+ private byte[][] blockletLevelMin, blockletLevelMax;
+
+ private Map<Integer, BlockletMinMax> blockMinMaxMap;
+
+ private String blockPath;
+
+
+ @Override public void onBlockStart(String blockId, String blockPath) {
+ pageLevelMax = null;
+ pageLevelMin = null;
+ blockletLevelMax = null;
+ blockletLevelMin = null;
+ blockMinMaxMap = null;
+ blockMinMaxMap = new HashMap<Integer, BlockletMinMax>();
+ this.blockPath = blockPath;
+ }
+
+ @Override public void onBlockEnd(String blockId) {
+ updateMinMaxIndex(blockId);
+ }
+
+ @Override public void onBlockletStart(int blockletId) {
+ }
+
+ @Override public void onBlockletEnd(int blockletId) {
+ updateBlockletMinMax(blockletId);
+ }
+
+ @Override
+ public void onPageAdded(int blockletId, int pageId, ColumnPage[] pages) {
+ // Calculate Min and Max value within this page.
+
+ // As part of example we are extracting Min Max values Manually. The same can be done from
+ // retrieving the page statistics. For e.g.
+
+ // if (pageLevelMin == null && pageLevelMax == null) {
+ // pageLevelMin[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMin());
+ // pageLevelMax[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMax());
+ // } else {
+ // if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMin[1], CarbonUtil
+ // .getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMin())) > 0) {
+ // pageLevelMin[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMin());
+ // }
+ // if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMax[1], CarbonUtil
+ // .getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMax())) < 0) {
+ // pageLevelMax[1] = CarbonUtil.getValueAsBytes(pages[0].getStatistics().getDataType(),
+ // pages[0].getStatistics().getMax());
+ // }
+
+ byte[] value = new byte[pages[0].getBytes(0).length - 2];
+ if (pageLevelMin == null && pageLevelMax == null) {
+ pageLevelMin = new byte[2][];
+ pageLevelMax = new byte[2][];
+
+ System.arraycopy(pages[0].getBytes(0), 2, value, 0, value.length);
+ pageLevelMin[1] = value;
+ pageLevelMax[1] = value;
+
+ } else {
+ for (int rowIndex = 0; rowIndex < pages[0].getPageSize(); rowIndex++) {
+ System.arraycopy(pages[0].getBytes(rowIndex), 2, value, 0, value.length);
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMin[1], value) > 0) {
+ pageLevelMin[1] = value;
+ }
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(pageLevelMax[1], value) < 0) {
+ pageLevelMax[1] = value;
+ }
+ }
+ }
+ }
+
+ private void updateBlockletMinMax(int blockletId) {
+ if (blockletLevelMax == null || blockletLevelMin == null) {
+ blockletLevelMax = new byte[2][];
+ blockletLevelMin = new byte[2][];
+ if (pageLevelMax != null || pageLevelMin != null) {
+ blockletLevelMin = pageLevelMin;
+ blockletLevelMax = pageLevelMax;
+ }
+ } else {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockletLevelMin[1], pageLevelMin[1]) > 0) {
+ blockletLevelMin = pageLevelMin;
+ }
+
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockletLevelMax[1], pageLevelMax[1]) > 0) {
+ blockletLevelMax = pageLevelMax;
+ }
+ }
+ BlockletMinMax blockletMinMax = new BlockletMinMax();
+ blockletMinMax.setMax(blockletLevelMax);
+ blockletMinMax.setMin(blockletLevelMin);
+ blockMinMaxMap.put(blockletId, blockletMinMax);
+ }
+
+
+ public void updateMinMaxIndex(String blockId) {
+ constructMinMaxIndex(blockId);
+ }
+
+
+
+ /**
+ * Construct the Min Max Index.
+ * @param blockId
+ */
+ public void constructMinMaxIndex(String blockId) {
+ // construct Min and Max values of each Blocklets present inside a block.
+ List<MinMaxIndexBlockDetails> tempMinMaxIndexBlockDetails = null;
+ tempMinMaxIndexBlockDetails = loadBlockDetails();
+ try {
+ writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockPath, blockId);
+ } catch (IOException ex) {
+ LOGGER.info(" Unable to write the file");
+ }
+ }
+
+ /**
+ * loadBlockDetails into the MinMaxIndexBlockDetails class.
+ */
+ private List<MinMaxIndexBlockDetails> loadBlockDetails() {
+ List<MinMaxIndexBlockDetails> minMaxIndexBlockDetails = new ArrayList<MinMaxIndexBlockDetails>();
+ MinMaxIndexBlockDetails tmpminMaxIndexBlockDetails = new MinMaxIndexBlockDetails();
+
+ for (int index = 0; index < blockMinMaxMap.size(); index++) {
+ tmpminMaxIndexBlockDetails.setMinValues(blockMinMaxMap.get(index).getMin());
+ tmpminMaxIndexBlockDetails.setMaxValues(blockMinMaxMap.get(index).getMax());
+ tmpminMaxIndexBlockDetails.setBlockletId(index);
+ tmpminMaxIndexBlockDetails.setFilePath(this.blockPath);
+ minMaxIndexBlockDetails.add(tmpminMaxIndexBlockDetails);
+ }
+ return minMaxIndexBlockDetails;
+ }
+
+ /**
+ * Write the data to a file. This is JSON format file.
+ * @param minMaxIndexBlockDetails
+ * @param blockPath
+ * @param blockId
+ * @throws IOException
+ */
+ public void writeMinMaxIndexFile(List<MinMaxIndexBlockDetails> minMaxIndexBlockDetails,
+ String blockPath, String blockId) throws IOException {
+ String filePath = blockPath.substring(0, blockPath.lastIndexOf(File.separator) + 1) + blockId
+ + ".minmaxindex";
+ BufferedWriter brWriter = null;
+ DataOutputStream dataOutStream = null;
+ try {
+ FileFactory.createNewFile(filePath, FileFactory.getFileType(filePath));
+ dataOutStream = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath));
+ Gson gsonObjectToWrite = new Gson();
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream,
+ CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT));
+ String minmaxIndexData = gsonObjectToWrite.toJson(minMaxIndexBlockDetails);
+ brWriter.write(minmaxIndexData);
+ } catch (IOException ioe) {
+ LOGGER.info("Error in writing minMaxindex file");
+ } finally {
+ if (null != brWriter) {
+ brWriter.flush();
+ }
+ if (null != dataOutStream) {
+ dataOutStream.flush();
+ }
+ CarbonUtil.closeStreams(brWriter, dataOutStream);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
new file mode 100644
index 0000000..0596db5
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.Serializable;
+
+public class MinMaxIndexBlockDetails implements Serializable {
+ private static final long serialVersionUID = 1206104914911491724L;
+
+ /**
+ * Min value of a column of one blocklet Bit-Packed
+ */
+ private byte[][] minValues;
+
+ /**
+ * Max value of a columns of one blocklet Bit-Packed
+ */
+ private byte[][] maxValues;
+
+ /**
+ * filePath pointing to the block.
+ */
+ private String filePath;
+
+ /**
+ * BlockletID of the block.
+ */
+ private Integer BlockletId;
+
+
+ public byte[][] getMinValues() {
+ return minValues;
+ }
+
+ public void setMinValues(byte[][] minValues) {
+ this.minValues = minValues;
+ }
+
+ public byte[][] getMaxValues() {
+ return maxValues;
+ }
+
+ public void setMaxValues(byte[][] maxValues) {
+ this.maxValues = maxValues;
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
+
+ public Integer getBlockletId() {
+ return BlockletId;
+ }
+
+ public void setBlockletId(Integer blockletId) {
+ BlockletId = blockletId;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala b/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
new file mode 100644
index 0000000..0cfe410
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.datamap.examples
+
+import java.io.File
+
+import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.util.CarbonProperties
+
+object MinMaxDataMapExample {
+ def main(args: Array[String]): Unit = {
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "").getCanonicalPath
+ val storeLocation = s"$rootPath/dataMap/examples/target/store"
+ val warehouse = s"$rootPath/datamap/examples/target/warehouse"
+ val metastoredb = s"$rootPath/datamap/examples/target"
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ import org.apache.spark.sql.CarbonSession._
+
+ val spark = SparkSession
+ .builder()
+ .master("local")
+ .appName("CarbonDataMapExample")
+ .config("spark.sql.warehouse.dir", warehouse)
+ .getOrCreateCarbonSession(storeLocation)
+
+ spark.sparkContext.setLogLevel("ERROR")
+ import spark.implicits._
+
+ // register datamap writer
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ AbsoluteTableIdentifier.from(storeLocation, "default", "carbonminmax"),
+ classOf[MinMaxDataMapFactory].getName,
+ MinMaxDataMap.NAME)
+
+ spark.sql("DROP TABLE IF EXISTS carbonminmax")
+
+ val df = spark.sparkContext.parallelize(1 to 33000)
+ .map(x => ("a", "b", x))
+ .toDF("c1", "c2", "c3")
+
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbonminmax")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ // Query the table.
+ spark.sql("select c2 from carbonminmax").show(20, false)
+ spark.sql("select c2 from carbonminmax where c2 = 'b'").show(20, false)
+ spark.sql("DROP TABLE IF EXISTS carbonminmax")
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index 74216ac..553e080 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -191,7 +191,7 @@ object DataMapWriterSuite {
callbackSeq :+= s"blocklet start $blockletId"
}
- override def onBlockStart(blockId: String): Unit = {
+ override def onBlockStart(blockId: String, blockPath: String): Unit = {
callbackSeq :+= s"block start $blockId"
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2399e26..66dc4ed 100644
--- a/pom.xml
+++ b/pom.xml
@@ -437,6 +437,7 @@
<module>format</module>
<module>integration/spark2</module>
<module>examples/spark2</module>
+ <module>datamap/examples</module>
<module>integration/hive</module>
<module>integration/presto</module>
<module>examples/flink</module>
@@ -473,6 +474,7 @@
<module>integration/presto</module>
<module>streaming</module>
<module>examples/spark2</module>
+ <module>datamap/examples</module>
</modules>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 4b0113c..8e350d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -82,10 +82,10 @@ public class DataMapWriterListener {
LOG.info("DataMapWriter " + writer + " added");
}
- public void onBlockStart(String blockId) {
+ public void onBlockStart(String blockId, String blockPath) {
for (List<DataMapWriter> writers : registry.values()) {
for (DataMapWriter writer : writers) {
- writer.onBlockStart(blockId);
+ writer.onBlockStart(blockId, blockPath);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index d1fc17b..7d0a285 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -269,7 +269,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
private void notifyDataMapBlockStart() {
if (listener != null) {
- listener.onBlockStart(carbonDataFileName);
+ listener.onBlockStart(carbonDataFileName, constructFactFileFullPath());
}
}
@@ -280,6 +280,11 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
blockletId = 0;
}
+ private String constructFactFileFullPath() {
+ String factFilePath =
+ this.dataWriterVo.getCarbonDataDirectoryPath() + File.separator + this.carbonDataFileName;
+ return factFilePath;
+ }
/**
* Finish writing current file. It will flush stream, copy and rename temp file to final file
* @param copyInCurrentThread set to false if want to do data copy in a new thread
http://git-wip-us.apache.org/repos/asf/carbondata/blob/cae74a8c/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index ddf444d..80d8154 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
import org.apache.carbondata.processing.store.TablePage;
import org.apache.carbondata.processing.store.writer.AbstractFactDataWriter;
+
/**
* Below class will be used to write the data in V3 format
* <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
@@ -157,6 +158,8 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter {
}
}
+
+
/**
* Write the collect blocklet data (blockletDataHolder) to file
*/
[09/50] [abbrv] carbondata git commit: [CARBONDATA-1987] Make package
name and directory paths consistent;
remove duplicate file CarbonColumnValidator
Posted by ra...@apache.org.
[CARBONDATA-1987] Make package name and directory paths consistent;remove duplicate file CarbonColumnValidator
add coveralls token to spark-2.2 profile;synchronize file path and package name;Delete duplicate class CarbonColumnValidator present in spark2 module
This closes #1764
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4d3f3989
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4d3f3989
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4d3f3989
Branch: refs/heads/fgdatamap
Commit: 4d3f3989b5b3aef6ed44e3c67c4102bea4505013
Parents: 94011c3
Author: Raghunandan S <ca...@gmail.com>
Authored: Thu Jan 4 20:18:07 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Jan 31 08:19:22 2018 +0530
----------------------------------------------------------------------
.../spark/CarbonColumnValidator.scala | 36 --
.../src/main/spark2.1/CarbonSQLConf.scala | 149 -------
.../src/main/spark2.1/CarbonSessionState.scala | 339 ----------------
.../apache/spark/sql/hive/CarbonSQLConf.scala | 149 +++++++
.../spark/sql/hive/CarbonSessionState.scala | 339 ++++++++++++++++
.../src/main/spark2.2/CarbonSessionState.scala | 398 -------------------
.../src/main/spark2.2/CarbonSqlConf.scala | 148 -------
.../spark/sql/hive/CarbonSessionState.scala | 398 +++++++++++++++++++
.../apache/spark/sql/hive/CarbonSqlConf.scala | 148 +++++++
pom.xml | 8 +-
10 files changed, 1039 insertions(+), 1073 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
deleted file mode 100644
index 03c4764..0000000
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark
-
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
- /**
- * Carbon column validator
- */
-class CarbonColumnValidator extends ColumnValidator {
- def validateColumns(allColumns: Seq[ColumnSchema]): Unit = {
- allColumns.foreach { columnSchema =>
- val colWithSameId = allColumns.filter { x =>
- x.getColumnUniqueId.equals(columnSchema.getColumnUniqueId)
- }
- if (colWithSameId.size > 1) {
- throw new MalformedCarbonCommandException("Two column can not have same columnId")
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala b/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
deleted file mode 100644
index 15ccb0c..0000000
--- a/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.internal.SQLConf.SQLConfigBuilder
-
-import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * To initialize dynamic values default param
- */
-class CarbonSQLConf(sparkSession: SparkSession) {
-
- val carbonProperties = CarbonProperties.getInstance()
-
- /**
- * To initialize dynamic param defaults along with usage docs
- */
- def addDefaultCarbonParams(): Unit = {
- val ENABLE_UNSAFE_SORT =
- SQLConfigBuilder(CarbonCommonConstants.ENABLE_UNSAFE_SORT)
- .doc("To enable/ disable unsafe sort.")
- .booleanConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
- val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
- SQLConfigBuilder(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
- .doc("To set carbon task distribution.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
- val BAD_RECORDS_LOGGER_ENABLE =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
- .doc("To enable/ disable carbon bad record logger.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants
- .CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
- val BAD_RECORDS_ACTION =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
- .doc("To configure the bad records action.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
- val IS_EMPTY_DATA_BAD_RECORD =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
- .doc("Property to decide weather empty data to be considered bad/ good record.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT
- .toBoolean)
- val SORT_SCOPE =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE)
- .doc("Property to specify sort scope.")
- .stringConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- val BATCH_SORT_SIZE_INMB =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB)
- .doc("Property to specify batch sort size in MB.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
- val SINGLE_PASS =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS)
- .doc("Property to enable/disable single_pass.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
- val BAD_RECORD_PATH =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH)
- .doc("Property to configure the bad record location.")
- .stringConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- val GLOBAL_SORT_PARTITIONS =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS)
- .doc("Property to configure the global sort partitions.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
- CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
- val DATEFORMAT =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
- .doc("Property to configure data format for date type columns.")
- .stringConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
- val CARBON_INPUT_SEGMENTS = SQLConfigBuilder(
- "carbon.input.segments.<database_name>.<table_name>")
- .doc("Property to configure the list of segments to query.").stringConf
- .createWithDefault(carbonProperties
- .getProperty("carbon.input.segments.<database_name>.<table_name>", "*"))
- }
- /**
- * to set the dynamic properties default values
- */
- def addDefaultCarbonSessionParams(): Unit = {
- sparkSession.conf.set(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
- sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
- CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
- CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
deleted file mode 100644
index 0fe0f96..0000000
--- a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
+++ /dev/null
@@ -1,339 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
-import org.apache.spark.sql.catalyst.catalog.{CatalogTablePartition, FunctionResourceLoader, GlobalTempViewManager, SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BoundReference, Expression, InterpretedPredicate, PredicateSubquery, ScalarSubquery}
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.ParserUtils._
-import org.apache.spark.sql.catalyst.parser.SqlBaseParser.CreateTableContext
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias}
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.{CatalystConf, TableIdentifier}
-import org.apache.spark.sql.execution.datasources._
-import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
-import org.apache.spark.sql.execution.{SparkOptimizer, SparkSqlAstBuilder}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
-import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlParser, CarbonSparkSqlParser}
-import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, ExperimentalMethods, SparkSession, Strategy}
-
-import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-/**
- * This class will have carbon catalog and refresh the relation from cache if the carbontable in
- * carbon catalog is not same as cached carbon relation's carbon table
- *
- * @param externalCatalog
- * @param globalTempViewManager
- * @param sparkSession
- * @param functionResourceLoader
- * @param functionRegistry
- * @param conf
- * @param hadoopConf
- */
-class CarbonSessionCatalog(
- externalCatalog: HiveExternalCatalog,
- globalTempViewManager: GlobalTempViewManager,
- sparkSession: SparkSession,
- functionResourceLoader: FunctionResourceLoader,
- functionRegistry: FunctionRegistry,
- conf: SQLConf,
- hadoopConf: Configuration)
- extends HiveSessionCatalog(
- externalCatalog,
- globalTempViewManager,
- sparkSession,
- functionResourceLoader,
- functionRegistry,
- conf,
- hadoopConf) {
-
- lazy val carbonEnv = {
- val env = new CarbonEnv
- env.init(sparkSession)
- env
- }
-
- // Initialize all listeners to the Operation bus.
- CarbonEnv.initListeners()
-
- /**
- * This method will invalidate carbonrelation from cache if carbon table is updated in
- * carbon catalog
- *
- * @param name
- * @param alias
- * @return
- */
- override def lookupRelation(name: TableIdentifier,
- alias: Option[String]): LogicalPlan = {
- val rtnRelation = super.lookupRelation(name, alias)
- var toRefreshRelation = false
- rtnRelation match {
- case SubqueryAlias(_,
- LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _), _) =>
- toRefreshRelation = refreshRelationFromCache(name, alias, carbonDatasourceHadoopRelation)
- case LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _) =>
- toRefreshRelation = refreshRelationFromCache(name, alias, carbonDatasourceHadoopRelation)
- case _ =>
- }
-
- if (toRefreshRelation) {
- super.lookupRelation(name, alias)
- } else {
- rtnRelation
- }
- }
-
- private def refreshRelationFromCache(identifier: TableIdentifier,
- alias: Option[String],
- carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation): Boolean = {
- var isRefreshed = false
- val storePath = CarbonProperties.getStorePath
- carbonEnv.carbonMetastore.
- checkSchemasModifiedTimeAndReloadTable(identifier)
-
- val table = carbonEnv.carbonMetastore.getTableFromMetadataCache(
- carbonDatasourceHadoopRelation.carbonTable.getDatabaseName,
- carbonDatasourceHadoopRelation.carbonTable.getTableName)
- if (table.isEmpty || (table.isDefined &&
- table.get.getTableLastUpdatedTime !=
- carbonDatasourceHadoopRelation.carbonTable.getTableLastUpdatedTime)) {
- refreshTable(identifier)
- DataMapStoreManager.getInstance().
- clearDataMaps(AbsoluteTableIdentifier.from(storePath,
- identifier.database.getOrElse("default"), identifier.table))
- isRefreshed = true
- logInfo(s"Schema changes have been detected for table: $identifier")
- }
- isRefreshed
- }
-
- /**
- * returns hive client from session state
- *
- * @return
- */
- def getClient(): org.apache.spark.sql.hive.client.HiveClient = {
- sparkSession.sessionState.asInstanceOf[CarbonSessionState].metadataHive
- }
-
- override def createPartitions(
- tableName: TableIdentifier,
- parts: Seq[CatalogTablePartition],
- ignoreIfExists: Boolean): Unit = {
- try {
- val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
- // Get the properties from thread local
- val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
- if (carbonSessionInfo != null) {
- val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
- super.createPartitions(tableName, updatedParts, ignoreIfExists)
- } else {
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
- } catch {
- case e: Exception =>
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
- }
-
- /**
- * This is alternate way of getting partition information. It first fetches all partitions from
- * hive and then apply filter instead of querying hive along with filters.
- * @param partitionFilters
- * @param sparkSession
- * @param identifier
- * @return
- */
- def getPartitionsAlternate(
- partitionFilters: Seq[Expression],
- sparkSession: SparkSession,
- identifier: TableIdentifier) = {
- val allPartitions = sparkSession.sessionState.catalog.listPartitions(identifier)
- val catalogTable = sparkSession.sessionState.catalog.getTableMetadata(identifier)
- val partitionSchema = catalogTable.partitionSchema
- if (partitionFilters.nonEmpty) {
- val boundPredicate =
- InterpretedPredicate.create(partitionFilters.reduce(And).transform {
- case att: AttributeReference =>
- val index = partitionSchema.indexWhere(_.name == att.name)
- BoundReference(index, partitionSchema(index).dataType, nullable = true)
- })
- allPartitions.filter { p => boundPredicate(p.toRow(partitionSchema)) }
- } else {
- allPartitions
- }
- }
-}
-
-/**
- * Session state implementation to override sql parser and adding strategies
- * @param sparkSession
- */
-class CarbonSessionState(sparkSession: SparkSession) extends HiveSessionState(sparkSession) {
-
- override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
-
- experimentalMethods.extraStrategies = extraStrategies
-
- experimentalMethods.extraOptimizations = extraOptimizations
-
- def extraStrategies: Seq[Strategy] = {
- Seq(
- new StreamingTableStrategy(sparkSession),
- new CarbonLateDecodeStrategy,
- new DDLStrategy(sparkSession)
- )
- }
-
- def extraOptimizations: Seq[Rule[LogicalPlan]] = {
- Seq(new CarbonIUDRule,
- new CarbonUDFTransformRule,
- new CarbonLateDecodeRule)
- }
-
- override lazy val optimizer: Optimizer = new CarbonOptimizer(catalog, conf, experimentalMethods)
-
- def extendedAnalyzerRules: Seq[Rule[LogicalPlan]] = Nil
- def internalAnalyzerRules: Seq[Rule[LogicalPlan]] = {
- catalog.ParquetConversions ::
- catalog.OrcConversions ::
- CarbonPreInsertionCasts(sparkSession) ::
- CarbonIUDAnalysisRule(sparkSession) ::
- AnalyzeCreateTable(sparkSession) ::
- PreprocessTableInsertion(conf) ::
- DataSourceAnalysis(conf) ::
- (if (conf.runSQLonFile) {
- new ResolveDataSource(sparkSession) :: Nil
- } else { Nil })
- }
-
- override lazy val analyzer: Analyzer =
- new CarbonAnalyzer(catalog, conf, sparkSession,
- new Analyzer(catalog, conf) {
- override val extendedResolutionRules =
- if (extendedAnalyzerRules.nonEmpty) {
- extendedAnalyzerRules ++ internalAnalyzerRules
- } else {
- internalAnalyzerRules
- }
- override val extendedCheckRules = Seq(
- PreWriteCheck(conf, catalog))
- }
- )
-
- /**
- * Internal catalog for managing table and database states.
- */
- override lazy val catalog = {
- new CarbonSessionCatalog(
- sparkSession.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog],
- sparkSession.sharedState.globalTempViewManager,
- sparkSession,
- functionResourceLoader,
- functionRegistry,
- conf,
- newHadoopConf())
- }
-}
-
-class CarbonAnalyzer(catalog: SessionCatalog,
- conf: CatalystConf,
- sparkSession: SparkSession,
- analyzer: Analyzer) extends Analyzer(catalog, conf) {
- override def execute(plan: LogicalPlan): LogicalPlan = {
- var logicalPlan = analyzer.execute(plan)
- logicalPlan = CarbonPreAggregateDataLoadingRules(sparkSession).apply(logicalPlan)
- CarbonPreAggregateQueryRules(sparkSession).apply(logicalPlan)
- }
-}
-
-class CarbonOptimizer(
- catalog: SessionCatalog,
- conf: SQLConf,
- experimentalMethods: ExperimentalMethods)
- extends SparkOptimizer(catalog, conf, experimentalMethods) {
-
- override def execute(plan: LogicalPlan): LogicalPlan = {
- val transFormedPlan: LogicalPlan = CarbonOptimizerUtil.transformForScalarSubQuery(plan)
- super.execute(transFormedPlan)
- }
-}
-
-object CarbonOptimizerUtil {
- def transformForScalarSubQuery(plan: LogicalPlan) : LogicalPlan = {
- // In case scalar subquery add flag in relation to skip the decoder plan in optimizer rule,
- // And optimize whole plan at once.
- val transFormedPlan = plan.transform {
- case filter: Filter =>
- filter.transformExpressions {
- case s: ScalarSubquery =>
- val tPlan = s.plan.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- ScalarSubquery(tPlan, s.children, s.exprId)
- case p: PredicateSubquery =>
- val tPlan = p.plan.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- PredicateSubquery(tPlan, p.children, p.nullAware, p.exprId)
- }
- }
- transFormedPlan
- }
-}
-
-class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
- extends SparkSqlAstBuilder(conf) {
-
- val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
-
- override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = {
- val fileStorage = helper.getFileStorage(ctx.createFileFormat)
-
- if (fileStorage.equalsIgnoreCase("'carbondata'") ||
- fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
- helper.createCarbonTable(
- tableHeader = ctx.createTableHeader,
- skewSpecContext = ctx.skewSpec,
- bucketSpecContext = ctx.bucketSpec,
- partitionColumns = ctx.partitionColumns,
- columns = ctx.columns,
- tablePropertyList = ctx.tablePropertyList,
- locationSpecContext = ctx.locationSpec(),
- tableComment = Option(ctx.STRING()).map(string),
- ctas = ctx.AS,
- query = ctx.query)
- } else {
- super.visitCreateTable(ctx)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSQLConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSQLConf.scala b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSQLConf.scala
new file mode 100644
index 0000000..15ccb0c
--- /dev/null
+++ b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSQLConf.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.internal.SQLConf.SQLConfigBuilder
+
+import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * To initialize dynamic values default param
+ */
+class CarbonSQLConf(sparkSession: SparkSession) {
+
+ val carbonProperties = CarbonProperties.getInstance()
+
+ /**
+ * To initialize dynamic param defaults along with usage docs
+ */
+ def addDefaultCarbonParams(): Unit = {
+ val ENABLE_UNSAFE_SORT =
+ SQLConfigBuilder(CarbonCommonConstants.ENABLE_UNSAFE_SORT)
+ .doc("To enable/ disable unsafe sort.")
+ .booleanConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
+ val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
+ SQLConfigBuilder(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
+ .doc("To set carbon task distribution.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
+ val BAD_RECORDS_LOGGER_ENABLE =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
+ .doc("To enable/ disable carbon bad record logger.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants
+ .CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
+ val BAD_RECORDS_ACTION =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
+ .doc("To configure the bad records action.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
+ val IS_EMPTY_DATA_BAD_RECORD =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
+ .doc("Property to decide weather empty data to be considered bad/ good record.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT
+ .toBoolean)
+ val SORT_SCOPE =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE)
+ .doc("Property to specify sort scope.")
+ .stringConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
+ val BATCH_SORT_SIZE_INMB =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB)
+ .doc("Property to specify batch sort size in MB.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
+ val SINGLE_PASS =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS)
+ .doc("Property to enable/disable single_pass.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
+ val BAD_RECORD_PATH =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH)
+ .doc("Property to configure the bad record location.")
+ .stringConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ val GLOBAL_SORT_PARTITIONS =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS)
+ .doc("Property to configure the global sort partitions.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
+ CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
+ val DATEFORMAT =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
+ .doc("Property to configure data format for date type columns.")
+ .stringConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
+ val CARBON_INPUT_SEGMENTS = SQLConfigBuilder(
+ "carbon.input.segments.<database_name>.<table_name>")
+ .doc("Property to configure the list of segments to query.").stringConf
+ .createWithDefault(carbonProperties
+ .getProperty("carbon.input.segments.<database_name>.<table_name>", "*"))
+ }
+ /**
+ * to set the dynamic properties default values
+ */
+ def addDefaultCarbonSessionParams(): Unit = {
+ sparkSession.conf.set(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
+ sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
+ carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
+ CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
new file mode 100644
index 0000000..0fe0f96
--- /dev/null
+++ b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
+import org.apache.spark.sql.catalyst.catalog.{CatalogTablePartition, FunctionResourceLoader, GlobalTempViewManager, SessionCatalog}
+import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BoundReference, Expression, InterpretedPredicate, PredicateSubquery, ScalarSubquery}
+import org.apache.spark.sql.catalyst.optimizer.Optimizer
+import org.apache.spark.sql.catalyst.parser.ParserInterface
+import org.apache.spark.sql.catalyst.parser.ParserUtils._
+import org.apache.spark.sql.catalyst.parser.SqlBaseParser.CreateTableContext
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.{CatalystConf, TableIdentifier}
+import org.apache.spark.sql.execution.datasources._
+import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
+import org.apache.spark.sql.execution.{SparkOptimizer, SparkSqlAstBuilder}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
+import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlParser, CarbonSparkSqlParser}
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, ExperimentalMethods, SparkSession, Strategy}
+
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
+import org.apache.carbondata.spark.util.CarbonScalaUtil
+
+/**
+ * This class will have carbon catalog and refresh the relation from cache if the carbontable in
+ * carbon catalog is not same as cached carbon relation's carbon table
+ *
+ * @param externalCatalog
+ * @param globalTempViewManager
+ * @param sparkSession
+ * @param functionResourceLoader
+ * @param functionRegistry
+ * @param conf
+ * @param hadoopConf
+ */
+class CarbonSessionCatalog(
+ externalCatalog: HiveExternalCatalog,
+ globalTempViewManager: GlobalTempViewManager,
+ sparkSession: SparkSession,
+ functionResourceLoader: FunctionResourceLoader,
+ functionRegistry: FunctionRegistry,
+ conf: SQLConf,
+ hadoopConf: Configuration)
+ extends HiveSessionCatalog(
+ externalCatalog,
+ globalTempViewManager,
+ sparkSession,
+ functionResourceLoader,
+ functionRegistry,
+ conf,
+ hadoopConf) {
+
+ lazy val carbonEnv = {
+ val env = new CarbonEnv
+ env.init(sparkSession)
+ env
+ }
+
+ // Initialize all listeners to the Operation bus.
+ CarbonEnv.initListeners()
+
+ /**
+ * This method will invalidate carbonrelation from cache if carbon table is updated in
+ * carbon catalog
+ *
+ * @param name
+ * @param alias
+ * @return
+ */
+ override def lookupRelation(name: TableIdentifier,
+ alias: Option[String]): LogicalPlan = {
+ val rtnRelation = super.lookupRelation(name, alias)
+ var toRefreshRelation = false
+ rtnRelation match {
+ case SubqueryAlias(_,
+ LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _), _) =>
+ toRefreshRelation = refreshRelationFromCache(name, alias, carbonDatasourceHadoopRelation)
+ case LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _) =>
+ toRefreshRelation = refreshRelationFromCache(name, alias, carbonDatasourceHadoopRelation)
+ case _ =>
+ }
+
+ if (toRefreshRelation) {
+ super.lookupRelation(name, alias)
+ } else {
+ rtnRelation
+ }
+ }
+
+ private def refreshRelationFromCache(identifier: TableIdentifier,
+ alias: Option[String],
+ carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation): Boolean = {
+ var isRefreshed = false
+ val storePath = CarbonProperties.getStorePath
+ carbonEnv.carbonMetastore.
+ checkSchemasModifiedTimeAndReloadTable(identifier)
+
+ val table = carbonEnv.carbonMetastore.getTableFromMetadataCache(
+ carbonDatasourceHadoopRelation.carbonTable.getDatabaseName,
+ carbonDatasourceHadoopRelation.carbonTable.getTableName)
+ if (table.isEmpty || (table.isDefined &&
+ table.get.getTableLastUpdatedTime !=
+ carbonDatasourceHadoopRelation.carbonTable.getTableLastUpdatedTime)) {
+ refreshTable(identifier)
+ DataMapStoreManager.getInstance().
+ clearDataMaps(AbsoluteTableIdentifier.from(storePath,
+ identifier.database.getOrElse("default"), identifier.table))
+ isRefreshed = true
+ logInfo(s"Schema changes have been detected for table: $identifier")
+ }
+ isRefreshed
+ }
+
+ /**
+ * returns hive client from session state
+ *
+ * @return
+ */
+ def getClient(): org.apache.spark.sql.hive.client.HiveClient = {
+ sparkSession.sessionState.asInstanceOf[CarbonSessionState].metadataHive
+ }
+
+ override def createPartitions(
+ tableName: TableIdentifier,
+ parts: Seq[CatalogTablePartition],
+ ignoreIfExists: Boolean): Unit = {
+ try {
+ val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ // Get the properties from thread local
+ val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
+ if (carbonSessionInfo != null) {
+ val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
+ super.createPartitions(tableName, updatedParts, ignoreIfExists)
+ } else {
+ super.createPartitions(tableName, parts, ignoreIfExists)
+ }
+ } catch {
+ case e: Exception =>
+ super.createPartitions(tableName, parts, ignoreIfExists)
+ }
+ }
+
+ /**
+ * This is alternate way of getting partition information. It first fetches all partitions from
+ * hive and then apply filter instead of querying hive along with filters.
+ * @param partitionFilters
+ * @param sparkSession
+ * @param identifier
+ * @return
+ */
+ def getPartitionsAlternate(
+ partitionFilters: Seq[Expression],
+ sparkSession: SparkSession,
+ identifier: TableIdentifier) = {
+ val allPartitions = sparkSession.sessionState.catalog.listPartitions(identifier)
+ val catalogTable = sparkSession.sessionState.catalog.getTableMetadata(identifier)
+ val partitionSchema = catalogTable.partitionSchema
+ if (partitionFilters.nonEmpty) {
+ val boundPredicate =
+ InterpretedPredicate.create(partitionFilters.reduce(And).transform {
+ case att: AttributeReference =>
+ val index = partitionSchema.indexWhere(_.name == att.name)
+ BoundReference(index, partitionSchema(index).dataType, nullable = true)
+ })
+ allPartitions.filter { p => boundPredicate(p.toRow(partitionSchema)) }
+ } else {
+ allPartitions
+ }
+ }
+}
+
+/**
+ * Session state implementation to override sql parser and adding strategies
+ * @param sparkSession
+ */
+class CarbonSessionState(sparkSession: SparkSession) extends HiveSessionState(sparkSession) {
+
+ override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
+
+ experimentalMethods.extraStrategies = extraStrategies
+
+ experimentalMethods.extraOptimizations = extraOptimizations
+
+ def extraStrategies: Seq[Strategy] = {
+ Seq(
+ new StreamingTableStrategy(sparkSession),
+ new CarbonLateDecodeStrategy,
+ new DDLStrategy(sparkSession)
+ )
+ }
+
+ def extraOptimizations: Seq[Rule[LogicalPlan]] = {
+ Seq(new CarbonIUDRule,
+ new CarbonUDFTransformRule,
+ new CarbonLateDecodeRule)
+ }
+
+ override lazy val optimizer: Optimizer = new CarbonOptimizer(catalog, conf, experimentalMethods)
+
+ def extendedAnalyzerRules: Seq[Rule[LogicalPlan]] = Nil
+ def internalAnalyzerRules: Seq[Rule[LogicalPlan]] = {
+ catalog.ParquetConversions ::
+ catalog.OrcConversions ::
+ CarbonPreInsertionCasts(sparkSession) ::
+ CarbonIUDAnalysisRule(sparkSession) ::
+ AnalyzeCreateTable(sparkSession) ::
+ PreprocessTableInsertion(conf) ::
+ DataSourceAnalysis(conf) ::
+ (if (conf.runSQLonFile) {
+ new ResolveDataSource(sparkSession) :: Nil
+ } else { Nil })
+ }
+
+ override lazy val analyzer: Analyzer =
+ new CarbonAnalyzer(catalog, conf, sparkSession,
+ new Analyzer(catalog, conf) {
+ override val extendedResolutionRules =
+ if (extendedAnalyzerRules.nonEmpty) {
+ extendedAnalyzerRules ++ internalAnalyzerRules
+ } else {
+ internalAnalyzerRules
+ }
+ override val extendedCheckRules = Seq(
+ PreWriteCheck(conf, catalog))
+ }
+ )
+
+ /**
+ * Internal catalog for managing table and database states.
+ */
+ override lazy val catalog = {
+ new CarbonSessionCatalog(
+ sparkSession.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog],
+ sparkSession.sharedState.globalTempViewManager,
+ sparkSession,
+ functionResourceLoader,
+ functionRegistry,
+ conf,
+ newHadoopConf())
+ }
+}
+
+class CarbonAnalyzer(catalog: SessionCatalog,
+ conf: CatalystConf,
+ sparkSession: SparkSession,
+ analyzer: Analyzer) extends Analyzer(catalog, conf) {
+ override def execute(plan: LogicalPlan): LogicalPlan = {
+ var logicalPlan = analyzer.execute(plan)
+ logicalPlan = CarbonPreAggregateDataLoadingRules(sparkSession).apply(logicalPlan)
+ CarbonPreAggregateQueryRules(sparkSession).apply(logicalPlan)
+ }
+}
+
+class CarbonOptimizer(
+ catalog: SessionCatalog,
+ conf: SQLConf,
+ experimentalMethods: ExperimentalMethods)
+ extends SparkOptimizer(catalog, conf, experimentalMethods) {
+
+ override def execute(plan: LogicalPlan): LogicalPlan = {
+ val transFormedPlan: LogicalPlan = CarbonOptimizerUtil.transformForScalarSubQuery(plan)
+ super.execute(transFormedPlan)
+ }
+}
+
+object CarbonOptimizerUtil {
+ def transformForScalarSubQuery(plan: LogicalPlan) : LogicalPlan = {
+ // In case scalar subquery add flag in relation to skip the decoder plan in optimizer rule,
+ // And optimize whole plan at once.
+ val transFormedPlan = plan.transform {
+ case filter: Filter =>
+ filter.transformExpressions {
+ case s: ScalarSubquery =>
+ val tPlan = s.plan.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ ScalarSubquery(tPlan, s.children, s.exprId)
+ case p: PredicateSubquery =>
+ val tPlan = p.plan.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ PredicateSubquery(tPlan, p.children, p.nullAware, p.exprId)
+ }
+ }
+ transFormedPlan
+ }
+}
+
+class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
+ extends SparkSqlAstBuilder(conf) {
+
+ val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
+
+ override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = {
+ val fileStorage = helper.getFileStorage(ctx.createFileFormat)
+
+ if (fileStorage.equalsIgnoreCase("'carbondata'") ||
+ fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
+ helper.createCarbonTable(
+ tableHeader = ctx.createTableHeader,
+ skewSpecContext = ctx.skewSpec,
+ bucketSpecContext = ctx.bucketSpec,
+ partitionColumns = ctx.partitionColumns,
+ columns = ctx.columns,
+ tablePropertyList = ctx.tablePropertyList,
+ locationSpecContext = ctx.locationSpec(),
+ tableComment = Option(ctx.STRING()).map(string),
+ ctas = ctx.AS,
+ query = ctx.query)
+ } else {
+ super.visitCreateTable(ctx)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
deleted file mode 100644
index 3c151f0..0000000
--- a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
+++ /dev/null
@@ -1,398 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-
-import scala.collection.generic.SeqFactory
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
-import org.apache.spark.sql.catalyst.catalog._
-import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BoundReference, Exists, Expression, In, InterpretedPredicate, ListQuery, ScalarSubquery}
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.ParserUtils.string
-import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{AddTableColumnsContext, ChangeColumnContext, CreateHiveTableContext, CreateTableContext}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias}
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.command.schema.{CarbonAlterTableAddColumnCommand, CarbonAlterTableDataTypeChangeCommand}
-import org.apache.spark.sql.execution.datasources.{FindDataSourceTable, LogicalRelation, PreWriteCheck, ResolveSQLOnFile, _}
-import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
-import org.apache.spark.sql.execution.{SparkOptimizer, SparkSqlAstBuilder}
-import org.apache.spark.sql.hive.client.HiveClient
-import org.apache.spark.sql.internal.{SQLConf, SessionState}
-import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
-import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlParser, CarbonSparkSqlParser}
-import org.apache.spark.sql.types.DecimalType
-import org.apache.spark.util.CarbonReflectionUtils
-
-import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-/**
- * This class will have carbon catalog and refresh the relation from cache if the carbontable in
- * carbon catalog is not same as cached carbon relation's carbon table
- *
- * @param externalCatalog
- * @param globalTempViewManager
- * @param sparkSession
- * @param functionResourceLoader
- * @param functionRegistry
- * @param conf
- * @param hadoopConf
- */
-class CarbonSessionCatalog(
- externalCatalog: HiveExternalCatalog,
- globalTempViewManager: GlobalTempViewManager,
- functionRegistry: FunctionRegistry,
- sparkSession: SparkSession,
- conf: SQLConf,
- hadoopConf: Configuration,
- parser: ParserInterface,
- functionResourceLoader: FunctionResourceLoader)
- extends HiveSessionCatalog(
- externalCatalog,
- globalTempViewManager,
- new HiveMetastoreCatalog(sparkSession),
- functionRegistry,
- conf,
- hadoopConf,
- parser,
- functionResourceLoader
- ) {
-
- lazy val carbonEnv = {
- val env = new CarbonEnv
- env.init(sparkSession)
- env
- }
-
- def getCarbonEnv() : CarbonEnv = {
- carbonEnv
- }
-
- // Initialize all listeners to the Operation bus.
- CarbonEnv.initListeners()
-
-
-
-
- override def lookupRelation(name: TableIdentifier): LogicalPlan = {
- val rtnRelation = super.lookupRelation(name)
- var toRefreshRelation = false
- rtnRelation match {
- case SubqueryAlias(_,
- LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
- toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
- case LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _) =>
- toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
- case SubqueryAlias(_, relation) if
- relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
- relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.HiveTableRelation") ||
- relation.getClass.getName.equals(
- "org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation") =>
- val catalogTable =
- CarbonReflectionUtils.getFieldOfCatalogTable(
- "tableMeta",
- relation).asInstanceOf[CatalogTable]
- toRefreshRelation =
- CarbonEnv.refreshRelationFromCache(catalogTable.identifier)(sparkSession)
- case _ =>
- }
-
- if (toRefreshRelation) {
- super.lookupRelation(name)
- } else {
- rtnRelation
- }
- }
-
- /**
- * returns hive client from HiveExternalCatalog
- *
- * @return
- */
- def getClient(): org.apache.spark.sql.hive.client.HiveClient = {
- sparkSession.asInstanceOf[CarbonSession].sharedState.externalCatalog
- .asInstanceOf[HiveExternalCatalog].client
- }
-
- override def createPartitions(
- tableName: TableIdentifier,
- parts: Seq[CatalogTablePartition],
- ignoreIfExists: Boolean): Unit = {
- try {
- val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
- // Get the properties from thread local
- val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
- if (carbonSessionInfo != null) {
- val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
- super.createPartitions(tableName, updatedParts, ignoreIfExists)
- } else {
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
- } catch {
- case e: Exception =>
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
- }
-
- /**
- * This is alternate way of getting partition information. It first fetches all partitions from
- * hive and then apply filter instead of querying hive along with filters.
- * @param partitionFilters
- * @param sparkSession
- * @param identifier
- * @return
- */
- def getPartitionsAlternate(partitionFilters: Seq[Expression],
- sparkSession: SparkSession,
- identifier: TableIdentifier) = {
- val allPartitions = sparkSession.sessionState.catalog.listPartitions(identifier)
- ExternalCatalogUtils.prunePartitionsByFilter(
- sparkSession.sessionState.catalog.getTableMetadata(identifier),
- allPartitions,
- partitionFilters,
- sparkSession.sessionState.conf.sessionLocalTimeZone)
- }
-}
-
-
-class CarbonAnalyzer(catalog: SessionCatalog,
- conf: SQLConf,
- sparkSession: SparkSession,
- analyzer: Analyzer) extends Analyzer(catalog, conf) {
- override def execute(plan: LogicalPlan): LogicalPlan = {
- var logicalPlan = analyzer.execute(plan)
- logicalPlan = CarbonPreAggregateDataLoadingRules(sparkSession).apply(logicalPlan)
- CarbonPreAggregateQueryRules(sparkSession).apply(logicalPlan)
- }
-}
-
-
-/**
- * Session state implementation to override sql parser and adding strategies
- *
- * @param sparkSession
- */
-class CarbonSessionStateBuilder(sparkSession: SparkSession,
- parentState: Option[SessionState] = None)
- extends HiveSessionStateBuilder(sparkSession, parentState) {
-
- override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
-
- experimentalMethods.extraStrategies =
- Seq(new StreamingTableStrategy(sparkSession),
- new CarbonLateDecodeStrategy,
- new DDLStrategy(sparkSession)
- )
- experimentalMethods.extraOptimizations = Seq(new CarbonIUDRule,
- new CarbonUDFTransformRule,
- new CarbonLateDecodeRule)
-
- /**
- * Internal catalog for managing table and database states.
- */
- /**
- * Create a [[CarbonSessionCatalogBuild]].
- */
- override protected lazy val catalog: CarbonSessionCatalog = {
- val catalog = new CarbonSessionCatalog(
- externalCatalog,
- session.sharedState.globalTempViewManager,
- functionRegistry,
- sparkSession,
- conf,
- SessionState.newHadoopConf(session.sparkContext.hadoopConfiguration, conf),
- sqlParser,
- resourceLoader)
- parentState.foreach(_.catalog.copyStateTo(catalog))
- catalog
- }
-
- private def externalCatalog: HiveExternalCatalog =
- session.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog]
-
- /**
- * Create a Hive aware resource loader.
- */
- override protected lazy val resourceLoader: HiveSessionResourceLoader = {
- val client: HiveClient = externalCatalog.client.newSession()
- new HiveSessionResourceLoader(session, client)
- }
-
- override lazy val optimizer: Optimizer = new CarbonOptimizer(catalog, conf, experimentalMethods)
-
- override protected def analyzer: Analyzer = new CarbonAnalyzer(catalog, conf, sparkSession,
- new Analyzer(catalog, conf) {
-
- override val extendedResolutionRules: Seq[Rule[LogicalPlan]] =
- new ResolveHiveSerdeTable(session) +:
- new FindDataSourceTable(session) +:
- new ResolveSQLOnFile(session) +:
- new CarbonIUDAnalysisRule(sparkSession) +:
- new CarbonPreInsertionCasts(sparkSession) +: customResolutionRules
-
- override val extendedCheckRules: Seq[LogicalPlan => Unit] =
- PreWriteCheck :: HiveOnlyCheck :: Nil
-
- override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
- new DetermineTableStats(session) +:
- RelationConversions(conf, catalog) +:
- PreprocessTableCreation(session) +:
- PreprocessTableInsertion(conf) +:
- DataSourceAnalysis(conf) +:
- HiveAnalysis +:
- customPostHocResolutionRules
- }
- )
-
- override protected def newBuilder: NewBuilder = new CarbonSessionStateBuilder(_, _)
-
-}
-
-
-class CarbonOptimizer(
- catalog: SessionCatalog,
- conf: SQLConf,
- experimentalMethods: ExperimentalMethods)
- extends SparkOptimizer(catalog, conf, experimentalMethods) {
-
- override def execute(plan: LogicalPlan): LogicalPlan = {
- // In case scalar subquery add flag in relation to skip the decoder plan in optimizer rule, And
- // optimize whole plan at once.
- val transFormedPlan = plan.transform {
- case filter: Filter =>
- filter.transformExpressions {
- case s: ScalarSubquery =>
- val tPlan = s.plan.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- ScalarSubquery(tPlan, s.children, s.exprId)
- case e: Exists =>
- val tPlan = e.plan.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- Exists(tPlan, e.children.map(_.canonicalized), e.exprId)
-
- case In(value, Seq(l@ListQuery(sub, _, exprId))) =>
- val tPlan = sub.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- In(value, Seq(ListQuery(tPlan, l.children , exprId)))
- }
- }
- super.execute(transFormedPlan)
- }
-}
-
-class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
- extends SparkSqlAstBuilder(conf) {
-
- val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
-
- override def visitCreateHiveTable(ctx: CreateHiveTableContext): LogicalPlan = {
- val fileStorage = helper.getFileStorage(ctx.createFileFormat)
-
- if (fileStorage.equalsIgnoreCase("'carbondata'") ||
- fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
- helper.createCarbonTable(
- tableHeader = ctx.createTableHeader,
- skewSpecContext = ctx.skewSpec,
- bucketSpecContext = ctx.bucketSpec,
- partitionColumns = ctx.partitionColumns,
- columns = ctx.columns,
- tablePropertyList = ctx.tablePropertyList,
- locationSpecContext = ctx.locationSpec(),
- tableComment = Option(ctx.STRING()).map(string),
- ctas = ctx.AS,
- query = ctx.query)
- } else {
- super.visitCreateHiveTable(ctx)
- }
- }
-
- override def visitChangeColumn(ctx: ChangeColumnContext): LogicalPlan = {
-
- val newColumn = visitColType(ctx.colType)
- if (!ctx.identifier.getText.equalsIgnoreCase(newColumn.name)) {
- throw new MalformedCarbonCommandException(
- "Column names provided are different. Both the column names should be same")
- }
-
- val (typeString, values) : (String, Option[List[(Int, Int)]]) = newColumn.dataType match {
- case d:DecimalType => ("decimal", Some(List((d.precision, d.scale))))
- case _ => (newColumn.dataType.typeName.toLowerCase, None)
- }
-
- val alterTableChangeDataTypeModel =
- AlterTableDataTypeChangeModel(new CarbonSpark2SqlParser().parseDataType(typeString, values),
- new CarbonSpark2SqlParser()
- .convertDbNameToLowerCase(Option(ctx.tableIdentifier().db).map(_.getText)),
- ctx.tableIdentifier().table.getText.toLowerCase,
- ctx.identifier.getText.toLowerCase,
- newColumn.name.toLowerCase)
-
- CarbonAlterTableDataTypeChangeCommand(alterTableChangeDataTypeModel)
- }
-
-
- override def visitAddTableColumns(ctx: AddTableColumnsContext): LogicalPlan = {
-
- val cols = Option(ctx.columns).toSeq.flatMap(visitColTypeList)
- val fields = parser.getFields(cols)
- val tblProperties = scala.collection.mutable.Map.empty[String, String]
- val tableModel = new CarbonSpark2SqlParser().prepareTableModel (false,
- new CarbonSpark2SqlParser().convertDbNameToLowerCase(Option(ctx.tableIdentifier().db)
- .map(_.getText)),
- ctx.tableIdentifier.table.getText.toLowerCase,
- fields,
- Seq.empty,
- tblProperties,
- None,
- true)
-
- val alterTableAddColumnsModel = AlterTableAddColumnsModel(
- Option(ctx.tableIdentifier().db).map(_.getText),
- ctx.tableIdentifier.table.getText,
- tblProperties.toMap,
- tableModel.dimCols,
- tableModel.msrCols,
- tableModel.highcardinalitydims.getOrElse(Seq.empty))
-
- CarbonAlterTableAddColumnCommand(alterTableAddColumnsModel)
- }
-
- override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = {
- super.visitCreateTable(ctx)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala b/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
deleted file mode 100644
index 2128ffd..0000000
--- a/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.internal.SQLConf.buildConf
-
-import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * To initialize dynamic values default param
- */
-class CarbonSQLConf(sparkSession: SparkSession) {
-
- val carbonProperties = CarbonProperties.getInstance()
-
- /**
- * To initialize dynamic param defaults along with usage docs
- */
- def addDefaultCarbonParams(): Unit = {
- val ENABLE_UNSAFE_SORT =
- buildConf(CarbonCommonConstants.ENABLE_UNSAFE_SORT)
- .doc("To enable/ disable unsafe sort.")
- .booleanConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
- val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
- buildConf(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
- .doc("To set carbon task distribution.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
- val BAD_RECORDS_LOGGER_ENABLE =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
- .doc("To enable/ disable carbon bad record logger.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants
- .CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
- val BAD_RECORDS_ACTION =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
- .doc("To configure the bad records action.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
- val IS_EMPTY_DATA_BAD_RECORD =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
- .doc("Property to decide weather empty data to be considered bad/ good record.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT
- .toBoolean)
- val SORT_SCOPE =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE)
- .doc("Property to specify sort scope.")
- .stringConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- val BATCH_SORT_SIZE_INMB =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB)
- .doc("Property to specify batch sort size in MB.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
- val SINGLE_PASS =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS)
- .doc("Property to enable/disable single_pass.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
- val BAD_RECORD_PATH =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH)
- .doc("Property to configure the bad record location.")
- .stringConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- val GLOBAL_SORT_PARTITIONS =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS)
- .doc("Property to configure the global sort partitions.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
- CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
- val DATEFORMAT =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
- .doc("Property to configure data format for date type columns.")
- .stringConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
- val CARBON_INPUT_SEGMENTS = buildConf(
- "carbon.input.segments.<database_name>.<table_name>")
- .doc("Property to configure the list of segments to query.").stringConf
- .createWithDefault(carbonProperties
- .getProperty("carbon.input.segments.<database_name>.<table_name>", "*"))
- }
- /**
- * to set the dynamic properties default values
- */
- def addDefaultCarbonSessionParams(): Unit = {
- sparkSession.conf.set(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
- sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
- CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
- CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
new file mode 100644
index 0000000..3c151f0
--- /dev/null
+++ b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+
+import scala.collection.generic.SeqFactory
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
+import org.apache.spark.sql.catalyst.catalog._
+import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BoundReference, Exists, Expression, In, InterpretedPredicate, ListQuery, ScalarSubquery}
+import org.apache.spark.sql.catalyst.optimizer.Optimizer
+import org.apache.spark.sql.catalyst.parser.ParserInterface
+import org.apache.spark.sql.catalyst.parser.ParserUtils.string
+import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{AddTableColumnsContext, ChangeColumnContext, CreateHiveTableContext, CreateTableContext}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.command._
+import org.apache.spark.sql.execution.command.schema.{CarbonAlterTableAddColumnCommand, CarbonAlterTableDataTypeChangeCommand}
+import org.apache.spark.sql.execution.datasources.{FindDataSourceTable, LogicalRelation, PreWriteCheck, ResolveSQLOnFile, _}
+import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
+import org.apache.spark.sql.execution.{SparkOptimizer, SparkSqlAstBuilder}
+import org.apache.spark.sql.hive.client.HiveClient
+import org.apache.spark.sql.internal.{SQLConf, SessionState}
+import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
+import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlParser, CarbonSparkSqlParser}
+import org.apache.spark.sql.types.DecimalType
+import org.apache.spark.util.CarbonReflectionUtils
+
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.spark.util.CarbonScalaUtil
+
+/**
+ * This class will have carbon catalog and refresh the relation from cache if the carbontable in
+ * carbon catalog is not same as cached carbon relation's carbon table
+ *
+ * @param externalCatalog
+ * @param globalTempViewManager
+ * @param sparkSession
+ * @param functionResourceLoader
+ * @param functionRegistry
+ * @param conf
+ * @param hadoopConf
+ */
+class CarbonSessionCatalog(
+ externalCatalog: HiveExternalCatalog,
+ globalTempViewManager: GlobalTempViewManager,
+ functionRegistry: FunctionRegistry,
+ sparkSession: SparkSession,
+ conf: SQLConf,
+ hadoopConf: Configuration,
+ parser: ParserInterface,
+ functionResourceLoader: FunctionResourceLoader)
+ extends HiveSessionCatalog(
+ externalCatalog,
+ globalTempViewManager,
+ new HiveMetastoreCatalog(sparkSession),
+ functionRegistry,
+ conf,
+ hadoopConf,
+ parser,
+ functionResourceLoader
+ ) {
+
+ lazy val carbonEnv = {
+ val env = new CarbonEnv
+ env.init(sparkSession)
+ env
+ }
+
+ def getCarbonEnv() : CarbonEnv = {
+ carbonEnv
+ }
+
+ // Initialize all listeners to the Operation bus.
+ CarbonEnv.initListeners()
+
+
+
+
+ override def lookupRelation(name: TableIdentifier): LogicalPlan = {
+ val rtnRelation = super.lookupRelation(name)
+ var toRefreshRelation = false
+ rtnRelation match {
+ case SubqueryAlias(_,
+ LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
+ toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
+ case LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _) =>
+ toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
+ case SubqueryAlias(_, relation) if
+ relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
+ relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.HiveTableRelation") ||
+ relation.getClass.getName.equals(
+ "org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation") =>
+ val catalogTable =
+ CarbonReflectionUtils.getFieldOfCatalogTable(
+ "tableMeta",
+ relation).asInstanceOf[CatalogTable]
+ toRefreshRelation =
+ CarbonEnv.refreshRelationFromCache(catalogTable.identifier)(sparkSession)
+ case _ =>
+ }
+
+ if (toRefreshRelation) {
+ super.lookupRelation(name)
+ } else {
+ rtnRelation
+ }
+ }
+
+ /**
+ * returns hive client from HiveExternalCatalog
+ *
+ * @return
+ */
+ def getClient(): org.apache.spark.sql.hive.client.HiveClient = {
+ sparkSession.asInstanceOf[CarbonSession].sharedState.externalCatalog
+ .asInstanceOf[HiveExternalCatalog].client
+ }
+
+ override def createPartitions(
+ tableName: TableIdentifier,
+ parts: Seq[CatalogTablePartition],
+ ignoreIfExists: Boolean): Unit = {
+ try {
+ val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ // Get the properties from thread local
+ val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
+ if (carbonSessionInfo != null) {
+ val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
+ super.createPartitions(tableName, updatedParts, ignoreIfExists)
+ } else {
+ super.createPartitions(tableName, parts, ignoreIfExists)
+ }
+ } catch {
+ case e: Exception =>
+ super.createPartitions(tableName, parts, ignoreIfExists)
+ }
+ }
+
+ /**
+ * This is alternate way of getting partition information. It first fetches all partitions from
+ * hive and then apply filter instead of querying hive along with filters.
+ * @param partitionFilters
+ * @param sparkSession
+ * @param identifier
+ * @return
+ */
+ def getPartitionsAlternate(partitionFilters: Seq[Expression],
+ sparkSession: SparkSession,
+ identifier: TableIdentifier) = {
+ val allPartitions = sparkSession.sessionState.catalog.listPartitions(identifier)
+ ExternalCatalogUtils.prunePartitionsByFilter(
+ sparkSession.sessionState.catalog.getTableMetadata(identifier),
+ allPartitions,
+ partitionFilters,
+ sparkSession.sessionState.conf.sessionLocalTimeZone)
+ }
+}
+
+
+class CarbonAnalyzer(catalog: SessionCatalog,
+ conf: SQLConf,
+ sparkSession: SparkSession,
+ analyzer: Analyzer) extends Analyzer(catalog, conf) {
+ override def execute(plan: LogicalPlan): LogicalPlan = {
+ var logicalPlan = analyzer.execute(plan)
+ logicalPlan = CarbonPreAggregateDataLoadingRules(sparkSession).apply(logicalPlan)
+ CarbonPreAggregateQueryRules(sparkSession).apply(logicalPlan)
+ }
+}
+
+
+/**
+ * Session state implementation to override sql parser and adding strategies
+ *
+ * @param sparkSession
+ */
+class CarbonSessionStateBuilder(sparkSession: SparkSession,
+ parentState: Option[SessionState] = None)
+ extends HiveSessionStateBuilder(sparkSession, parentState) {
+
+ override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
+
+ experimentalMethods.extraStrategies =
+ Seq(new StreamingTableStrategy(sparkSession),
+ new CarbonLateDecodeStrategy,
+ new DDLStrategy(sparkSession)
+ )
+ experimentalMethods.extraOptimizations = Seq(new CarbonIUDRule,
+ new CarbonUDFTransformRule,
+ new CarbonLateDecodeRule)
+
+ /**
+ * Internal catalog for managing table and database states.
+ */
+ /**
+ * Create a [[CarbonSessionCatalogBuild]].
+ */
+ override protected lazy val catalog: CarbonSessionCatalog = {
+ val catalog = new CarbonSessionCatalog(
+ externalCatalog,
+ session.sharedState.globalTempViewManager,
+ functionRegistry,
+ sparkSession,
+ conf,
+ SessionState.newHadoopConf(session.sparkContext.hadoopConfiguration, conf),
+ sqlParser,
+ resourceLoader)
+ parentState.foreach(_.catalog.copyStateTo(catalog))
+ catalog
+ }
+
+ private def externalCatalog: HiveExternalCatalog =
+ session.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog]
+
+ /**
+ * Create a Hive aware resource loader.
+ */
+ override protected lazy val resourceLoader: HiveSessionResourceLoader = {
+ val client: HiveClient = externalCatalog.client.newSession()
+ new HiveSessionResourceLoader(session, client)
+ }
+
+ override lazy val optimizer: Optimizer = new CarbonOptimizer(catalog, conf, experimentalMethods)
+
+ override protected def analyzer: Analyzer = new CarbonAnalyzer(catalog, conf, sparkSession,
+ new Analyzer(catalog, conf) {
+
+ override val extendedResolutionRules: Seq[Rule[LogicalPlan]] =
+ new ResolveHiveSerdeTable(session) +:
+ new FindDataSourceTable(session) +:
+ new ResolveSQLOnFile(session) +:
+ new CarbonIUDAnalysisRule(sparkSession) +:
+ new CarbonPreInsertionCasts(sparkSession) +: customResolutionRules
+
+ override val extendedCheckRules: Seq[LogicalPlan => Unit] =
+ PreWriteCheck :: HiveOnlyCheck :: Nil
+
+ override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
+ new DetermineTableStats(session) +:
+ RelationConversions(conf, catalog) +:
+ PreprocessTableCreation(session) +:
+ PreprocessTableInsertion(conf) +:
+ DataSourceAnalysis(conf) +:
+ HiveAnalysis +:
+ customPostHocResolutionRules
+ }
+ )
+
+ override protected def newBuilder: NewBuilder = new CarbonSessionStateBuilder(_, _)
+
+}
+
+
+class CarbonOptimizer(
+ catalog: SessionCatalog,
+ conf: SQLConf,
+ experimentalMethods: ExperimentalMethods)
+ extends SparkOptimizer(catalog, conf, experimentalMethods) {
+
+ override def execute(plan: LogicalPlan): LogicalPlan = {
+ // In case scalar subquery add flag in relation to skip the decoder plan in optimizer rule, And
+ // optimize whole plan at once.
+ val transFormedPlan = plan.transform {
+ case filter: Filter =>
+ filter.transformExpressions {
+ case s: ScalarSubquery =>
+ val tPlan = s.plan.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ ScalarSubquery(tPlan, s.children, s.exprId)
+ case e: Exists =>
+ val tPlan = e.plan.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ Exists(tPlan, e.children.map(_.canonicalized), e.exprId)
+
+ case In(value, Seq(l@ListQuery(sub, _, exprId))) =>
+ val tPlan = sub.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ In(value, Seq(ListQuery(tPlan, l.children , exprId)))
+ }
+ }
+ super.execute(transFormedPlan)
+ }
+}
+
+class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
+ extends SparkSqlAstBuilder(conf) {
+
+ val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
+
+ override def visitCreateHiveTable(ctx: CreateHiveTableContext): LogicalPlan = {
+ val fileStorage = helper.getFileStorage(ctx.createFileFormat)
+
+ if (fileStorage.equalsIgnoreCase("'carbondata'") ||
+ fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
+ helper.createCarbonTable(
+ tableHeader = ctx.createTableHeader,
+ skewSpecContext = ctx.skewSpec,
+ bucketSpecContext = ctx.bucketSpec,
+ partitionColumns = ctx.partitionColumns,
+ columns = ctx.columns,
+ tablePropertyList = ctx.tablePropertyList,
+ locationSpecContext = ctx.locationSpec(),
+ tableComment = Option(ctx.STRING()).map(string),
+ ctas = ctx.AS,
+ query = ctx.query)
+ } else {
+ super.visitCreateHiveTable(ctx)
+ }
+ }
+
+ override def visitChangeColumn(ctx: ChangeColumnContext): LogicalPlan = {
+
+ val newColumn = visitColType(ctx.colType)
+ if (!ctx.identifier.getText.equalsIgnoreCase(newColumn.name)) {
+ throw new MalformedCarbonCommandException(
+ "Column names provided are different. Both the column names should be same")
+ }
+
+ val (typeString, values) : (String, Option[List[(Int, Int)]]) = newColumn.dataType match {
+ case d:DecimalType => ("decimal", Some(List((d.precision, d.scale))))
+ case _ => (newColumn.dataType.typeName.toLowerCase, None)
+ }
+
+ val alterTableChangeDataTypeModel =
+ AlterTableDataTypeChangeModel(new CarbonSpark2SqlParser().parseDataType(typeString, values),
+ new CarbonSpark2SqlParser()
+ .convertDbNameToLowerCase(Option(ctx.tableIdentifier().db).map(_.getText)),
+ ctx.tableIdentifier().table.getText.toLowerCase,
+ ctx.identifier.getText.toLowerCase,
+ newColumn.name.toLowerCase)
+
+ CarbonAlterTableDataTypeChangeCommand(alterTableChangeDataTypeModel)
+ }
+
+
+ override def visitAddTableColumns(ctx: AddTableColumnsContext): LogicalPlan = {
+
+ val cols = Option(ctx.columns).toSeq.flatMap(visitColTypeList)
+ val fields = parser.getFields(cols)
+ val tblProperties = scala.collection.mutable.Map.empty[String, String]
+ val tableModel = new CarbonSpark2SqlParser().prepareTableModel (false,
+ new CarbonSpark2SqlParser().convertDbNameToLowerCase(Option(ctx.tableIdentifier().db)
+ .map(_.getText)),
+ ctx.tableIdentifier.table.getText.toLowerCase,
+ fields,
+ Seq.empty,
+ tblProperties,
+ None,
+ true)
+
+ val alterTableAddColumnsModel = AlterTableAddColumnsModel(
+ Option(ctx.tableIdentifier().db).map(_.getText),
+ ctx.tableIdentifier.table.getText,
+ tblProperties.toMap,
+ tableModel.dimCols,
+ tableModel.msrCols,
+ tableModel.highcardinalitydims.getOrElse(Seq.empty))
+
+ CarbonAlterTableAddColumnCommand(alterTableAddColumnsModel)
+ }
+
+ override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = {
+ super.visitCreateTable(ctx)
+ }
+}
[38/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index c33d5ac..1e73dbb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -43,10 +43,9 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNodeWrapper;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNode;
import org.apache.carbondata.core.indexstore.blockletindex.IndexWrapper;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -64,8 +63,8 @@ import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.SingleTableProvider;
import org.apache.carbondata.core.scan.filter.TableProvider;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
@@ -122,7 +121,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
queryProperties.queryStatisticsRecorder =
CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId());
queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
- QueryUtil.resolveQueryModel(queryModel);
QueryStatistic queryStatistic = new QueryStatistic();
// sort the block info
// so block will be loaded in sorted order this will be required for
@@ -169,12 +167,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
// calculating the total number of aggeragted columns
- int measureCount = queryModel.getQueryMeasures().size();
+ int measureCount = queryModel.getProjectionMeasures().size();
int currentIndex = 0;
DataType[] dataTypes = new DataType[measureCount];
- for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
+ for (ProjectionMeasure carbonMeasure : queryModel.getProjectionMeasures()) {
// adding the data type and aggregation type of all the measure this
// can be used
// to select the aggregator
@@ -199,9 +197,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
queryStatistic = new QueryStatistic();
// dictionary column unique column id to dictionary mapping
// which will be used to get column actual data
- queryProperties.columnToDictionayMapping = QueryUtil
- .getDimensionDictionaryDetail(queryModel.getQueryDimension(),
- queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier(),
+ queryProperties.columnToDictionayMapping =
+ QueryUtil.getDimensionDictionaryDetail(
+ queryModel.getProjectionDimensions(),
+ queryProperties.complexFilterDimension,
+ queryModel.getAbsoluteTableIdentifier(),
tableProvider);
queryStatistic
.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis());
@@ -253,8 +253,8 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// and query will be executed based on that infos
for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
AbstractIndex abstractIndex = queryProperties.dataBlocks.get(i);
- BlockletDataRefNodeWrapper dataRefNode =
- (BlockletDataRefNodeWrapper) abstractIndex.getDataRefNode();
+ BlockletDataRefNode dataRefNode =
+ (BlockletDataRefNode) abstractIndex.getDataRefNode();
blockExecutionInfoList.add(getBlockExecutionInfoForBlock(queryModel, abstractIndex,
dataRefNode.getBlockInfos().get(0).getBlockletInfos().getStartBlockletNumber(),
dataRefNode.numberOfNodes(), dataRefNode.getBlockInfos().get(0).getFilePath(),
@@ -278,20 +278,19 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
* @return block execution info
* @throws QueryExecutionException any failure during block info creation
*/
- protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
+ private BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath,
String[] deleteDeltaFiles)
throws QueryExecutionException {
BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
List<CarbonDimension> tableBlockDimensions = segmentProperties.getDimensions();
- KeyGenerator blockKeyGenerator = segmentProperties.getDimensionKeyGenerator();
// below is to get only those dimension in query which is present in the
// table block
- List<QueryDimension> currentBlockQueryDimensions = RestructureUtil
+ List<ProjectionDimension> projectDimensions = RestructureUtil
.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo,
- queryModel.getQueryDimension(), tableBlockDimensions,
+ queryModel.getProjectionDimensions(), tableBlockDimensions,
segmentProperties.getComplexDimensions());
int tableFactPathLength = CarbonStorePath
.getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getTablePath(),
@@ -301,27 +300,26 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
blockExecutionInfo.setDeleteDeltaFilePath(deleteDeltaFiles);
blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
- blockExecutionInfo.setQueryDimensions(currentBlockQueryDimensions
- .toArray(new QueryDimension[currentBlockQueryDimensions.size()]));
+ blockExecutionInfo.setProjectionDimensions(projectDimensions
+ .toArray(new ProjectionDimension[projectDimensions.size()]));
// get measures present in the current block
- List<QueryMeasure> currentBlockQueryMeasures =
+ List<ProjectionMeasure> currentBlockQueryMeasures =
getCurrentBlockQueryMeasures(blockExecutionInfo, queryModel, blockIndex);
- blockExecutionInfo.setQueryMeasures(
- currentBlockQueryMeasures.toArray(new QueryMeasure[currentBlockQueryMeasures.size()]));
+ blockExecutionInfo.setProjectionMeasures(
+ currentBlockQueryMeasures.toArray(new ProjectionMeasure[currentBlockQueryMeasures.size()]));
blockExecutionInfo.setDataBlock(blockIndex);
- blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
// setting whether raw record query or not
blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
// total number dimension
blockExecutionInfo
- .setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
+ .setTotalNumberDimensionToRead(
+ segmentProperties.getDimensionOrdinalToChunkMapping().size());
blockExecutionInfo.setPrefetchBlocklet(!queryModel.isReadPageByPage());
blockExecutionInfo
- .setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
- blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
+ .setTotalNumberOfMeasureToRead(segmentProperties.getMeasuresOrdinalToChunkMapping().size());
blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil
- .getComplexDimensionsMap(currentBlockQueryDimensions,
- segmentProperties.getDimensionOrdinalToBlockMapping(),
+ .getComplexDimensionsMap(projectDimensions,
+ segmentProperties.getDimensionOrdinalToChunkMapping(),
segmentProperties.getEachComplexDimColumnValueSize(),
queryProperties.columnToDictionayMapping, queryProperties.complexFilterDimension));
IndexKey startIndexKey = null;
@@ -355,45 +353,46 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// create a list of filter dimensions present in the current block
Set<CarbonDimension> currentBlockFilterDimensions =
getCurrentBlockFilterDimensions(queryProperties.complexFilterDimension, segmentProperties);
- int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(currentBlockQueryDimensions,
- segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions,
- currentBlockFilterDimensions, allProjectionListDimensionIdexes);
+ int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(
+ projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(),
+ expressionDimensions, currentBlockFilterDimensions, allProjectionListDimensionIdexes);
int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE));
- if (dimensionsBlockIndexes.length > 0) {
- numberOfElementToConsider = dimensionsBlockIndexes[dimensionsBlockIndexes.length - 1]
+ if (dimensionChunkIndexes.length > 0) {
+ numberOfElementToConsider = dimensionChunkIndexes[dimensionChunkIndexes.length - 1]
== segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ?
- dimensionsBlockIndexes.length - 1 :
- dimensionsBlockIndexes.length;
- blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(CarbonUtil
- .getRangeIndex(dimensionsBlockIndexes, numberOfElementToConsider,
+ dimensionChunkIndexes.length - 1 :
+ dimensionChunkIndexes.length;
+ blockExecutionInfo.setAllSelectedDimensionColumnIndexRange(
+ CarbonUtil.getRangeIndex(dimensionChunkIndexes, numberOfElementToConsider,
numberOfColumnToBeReadInOneIO));
} else {
- blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[0][0]);
+ blockExecutionInfo.setAllSelectedDimensionColumnIndexRange(new int[0][0]);
}
// get the list of updated filter measures present in the current block
- Set<CarbonMeasure> currentBlockFilterMeasures =
+ Set<CarbonMeasure> filterMeasures =
getCurrentBlockFilterMeasures(queryProperties.filterMeasures, segmentProperties);
// list of measures to be projected
List<Integer> allProjectionListMeasureIndexes = new ArrayList<>();
- int[] measureBlockIndexes = QueryUtil
- .getMeasureBlockIndexes(currentBlockQueryMeasures, expressionMeasures,
- segmentProperties.getMeasuresOrdinalToBlockMapping(), currentBlockFilterMeasures,
- allProjectionListMeasureIndexes);
- if (measureBlockIndexes.length > 0) {
+ int[] measureChunkIndexes = QueryUtil.getMeasureChunkIndexes(
+ currentBlockQueryMeasures, expressionMeasures,
+ segmentProperties.getMeasuresOrdinalToChunkMapping(), filterMeasures,
+ allProjectionListMeasureIndexes);
+ if (measureChunkIndexes.length > 0) {
- numberOfElementToConsider = measureBlockIndexes[measureBlockIndexes.length - 1]
+ numberOfElementToConsider = measureChunkIndexes[measureChunkIndexes.length - 1]
== segmentProperties.getMeasures().size() - 1 ?
- measureBlockIndexes.length - 1 :
- measureBlockIndexes.length;
+ measureChunkIndexes.length - 1 :
+ measureChunkIndexes.length;
// setting all the measure chunk indexes to be read from file
- blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(CarbonUtil
- .getRangeIndex(measureBlockIndexes, numberOfElementToConsider,
+ blockExecutionInfo.setAllSelectedMeasureIndexRange(
+ CarbonUtil.getRangeIndex(
+ measureChunkIndexes, numberOfElementToConsider,
numberOfColumnToBeReadInOneIO));
} else {
- blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[0][0]);
+ blockExecutionInfo.setAllSelectedMeasureIndexRange(new int[0][0]);
}
// setting the indexes of list of dimension in projection list
blockExecutionInfo.setProjectionListDimensionIndexes(ArrayUtils.toPrimitive(
@@ -405,27 +404,27 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
.toArray(new Integer[allProjectionListMeasureIndexes.size()])));
// setting the size of fixed key column (dictionary column)
blockExecutionInfo
- .setFixedLengthKeySize(getKeySize(currentBlockQueryDimensions, segmentProperties));
- Set<Integer> dictionaryColumnBlockIndex = new HashSet<Integer>();
- List<Integer> noDictionaryColumnBlockIndex = new ArrayList<Integer>();
+ .setFixedLengthKeySize(getKeySize(projectDimensions, segmentProperties));
+ Set<Integer> dictionaryColumnChunkIndex = new HashSet<Integer>();
+ List<Integer> noDictionaryColumnChunkIndex = new ArrayList<Integer>();
// get the block index to be read from file for query dimension
// for both dictionary columns and no dictionary columns
- QueryUtil.fillQueryDimensionsBlockIndexes(currentBlockQueryDimensions,
- segmentProperties.getDimensionOrdinalToBlockMapping(), dictionaryColumnBlockIndex,
- noDictionaryColumnBlockIndex);
- int[] queryDictionaryColumnBlockIndexes = ArrayUtils.toPrimitive(
- dictionaryColumnBlockIndex.toArray(new Integer[dictionaryColumnBlockIndex.size()]));
+ QueryUtil.fillQueryDimensionChunkIndexes(projectDimensions,
+ segmentProperties.getDimensionOrdinalToChunkMapping(), dictionaryColumnChunkIndex,
+ noDictionaryColumnChunkIndex);
+ int[] queryDictionaryColumnChunkIndexes = ArrayUtils.toPrimitive(
+ dictionaryColumnChunkIndex.toArray(new Integer[dictionaryColumnChunkIndex.size()]));
// need to sort the dictionary column as for all dimension
// column key will be filled based on key order
- Arrays.sort(queryDictionaryColumnBlockIndexes);
- blockExecutionInfo.setDictionaryColumnBlockIndex(queryDictionaryColumnBlockIndexes);
+ Arrays.sort(queryDictionaryColumnChunkIndexes);
+ blockExecutionInfo.setDictionaryColumnChunkIndex(queryDictionaryColumnChunkIndexes);
// setting the no dictionary column block indexes
- blockExecutionInfo.setNoDictionaryBlockIndexes(ArrayUtils.toPrimitive(
- noDictionaryColumnBlockIndex.toArray(new Integer[noDictionaryColumnBlockIndex.size()])));
+ blockExecutionInfo.setNoDictionaryColumnChunkIndexes(ArrayUtils.toPrimitive(
+ noDictionaryColumnChunkIndex.toArray(new Integer[noDictionaryColumnChunkIndex.size()])));
// setting each column value size
blockExecutionInfo.setEachColumnValueSize(segmentProperties.getEachDimColumnValueSize());
blockExecutionInfo.setComplexColumnParentBlockIndexes(
- getComplexDimensionParentBlockIndexes(currentBlockQueryDimensions));
+ getComplexDimensionParentBlockIndexes(projectDimensions));
blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
try {
// to set column group and its key structure info which will be used
@@ -433,15 +432,15 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// for getting the column group column data in case of final row
// and in case of dimension aggregation
blockExecutionInfo.setColumnGroupToKeyStructureInfo(
- QueryUtil.getColumnGroupKeyStructureInfo(currentBlockQueryDimensions, segmentProperties));
+ QueryUtil.getColumnGroupKeyStructureInfo(projectDimensions, segmentProperties));
} catch (KeyGenException e) {
throw new QueryExecutionException(e);
}
// set actual query dimensions and measures. It may differ in case of restructure scenarios
- blockExecutionInfo.setActualQueryDimensions(queryModel.getQueryDimension()
- .toArray(new QueryDimension[queryModel.getQueryDimension().size()]));
- blockExecutionInfo.setActualQueryMeasures(queryModel.getQueryMeasures()
- .toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
+ blockExecutionInfo.setActualQueryDimensions(queryModel.getProjectionDimensions()
+ .toArray(new ProjectionDimension[queryModel.getProjectionDimensions().size()]));
+ blockExecutionInfo.setActualQueryMeasures(queryModel.getProjectionMeasures()
+ .toArray(new ProjectionMeasure[queryModel.getProjectionMeasures().size()]));
DataTypeUtil.setDataTypeConverter(queryModel.getConverter());
return blockExecutionInfo;
}
@@ -454,7 +453,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
* @param blockMetadataInfo block metadata info
* @return key size
*/
- private int getKeySize(List<QueryDimension> queryDimension,
+ private int getKeySize(List<ProjectionDimension> queryDimension,
SegmentProperties blockMetadataInfo) {
// add the dimension block ordinal for each dictionary column
// existing in the current block dimensions. Set is used because in case of column groups
@@ -465,12 +464,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
while (counter < queryDimension.size()) {
if (queryDimension.get(counter).getDimension().getNumberOfChild() > 0) {
counter += queryDimension.get(counter).getDimension().getNumberOfChild();
- continue;
} else if (!CarbonUtil.hasEncoding(queryDimension.get(counter).getDimension().getEncoder(),
Encoding.DICTIONARY)) {
counter++;
} else {
- fixedLengthDimensionOrdinal.add(blockMetadataInfo.getDimensionOrdinalToBlockMapping()
+ fixedLengthDimensionOrdinal.add(blockMetadataInfo.getDimensionOrdinalToChunkMapping()
.get(queryDimension.get(counter).getDimension().getOrdinal()));
counter++;
}
@@ -492,26 +490,26 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
/**
* Below method will be used to get the measures present in the current block
*
- * @param blockExecutionInfo
+ * @param executionInfo
* @param queryModel query model
* @param tableBlock table block
* @return
*/
- private List<QueryMeasure> getCurrentBlockQueryMeasures(BlockExecutionInfo blockExecutionInfo,
+ private List<ProjectionMeasure> getCurrentBlockQueryMeasures(BlockExecutionInfo executionInfo,
QueryModel queryModel, AbstractIndex tableBlock) throws QueryExecutionException {
// getting the measure info which will be used while filling up measure data
- List<QueryMeasure> updatedQueryMeasures = RestructureUtil
- .createMeasureInfoAndGetCurrentBlockQueryMeasures(blockExecutionInfo,
- queryModel.getQueryMeasures(), tableBlock.getSegmentProperties().getMeasures());
+ List<ProjectionMeasure> updatedQueryMeasures = RestructureUtil
+ .createMeasureInfoAndGetCurrentBlockQueryMeasures(executionInfo,
+ queryModel.getProjectionMeasures(), tableBlock.getSegmentProperties().getMeasures());
// setting the measure aggregator for all aggregation function selected
// in query
- blockExecutionInfo.getMeasureInfo().setMeasureDataTypes(queryProperties.measureDataTypes);
+ executionInfo.getMeasureInfo().setMeasureDataTypes(queryProperties.measureDataTypes);
return updatedQueryMeasures;
}
- private int[] getComplexDimensionParentBlockIndexes(List<QueryDimension> queryDimensions) {
+ private int[] getComplexDimensionParentBlockIndexes(List<ProjectionDimension> queryDimensions) {
List<Integer> parentBlockIndexList = new ArrayList<Integer>();
- for (QueryDimension queryDimension : queryDimensions) {
+ for (ProjectionDimension queryDimension : queryDimensions) {
if (queryDimension.getDimension().getDataType().isComplexType()) {
parentBlockIndexList.add(queryDimension.getDimension().getOrdinal());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
index 931b1ab..93d696b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
@@ -23,7 +23,7 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.scan.result.iterator.DetailQueryResultIterator;
/**
@@ -31,10 +31,10 @@ import org.apache.carbondata.core.scan.result.iterator.DetailQueryResultIterator
* For executing the detail query it will pass all the block execution
* info to detail query result iterator and iterator will be returned
*/
-public class DetailQueryExecutor extends AbstractQueryExecutor<BatchResult> {
+public class DetailQueryExecutor extends AbstractQueryExecutor<RowBatch> {
@Override
- public CarbonIterator<BatchResult> execute(QueryModel queryModel)
+ public CarbonIterator<RowBatch> execute(QueryModel queryModel)
throws QueryExecutionException, IOException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
this.queryIterator = new DetailQueryResultIterator(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index 65f2838..06adf8f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -21,13 +21,11 @@ import java.util.Map;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
/**
* Below class will have all the properties which needed during query execution
@@ -41,14 +39,6 @@ public class BlockExecutionInfo {
private AbstractIndex blockIndex;
/**
- * each segment key size can be different and in that case we need to update
- * the fixed key with latest segment key generator. so this property will
- * tell whether this is required or not if key size is same then it is not
- * required
- */
- private boolean isFixedKeyUpdateRequired;
-
- /**
* below to store all the information required for measures during query
* execution
*/
@@ -73,22 +63,22 @@ public class BlockExecutionInfo {
/**
* total number of dimension in block
*/
- private int totalNumberDimensionBlock;
+ private int totalNumberDimensionToRead;
/**
* total number of measure in block
*/
- private int totalNumberOfMeasureBlock;
+ private int totalNumberOfMeasureToRead;
/**
* will be used to read the dimension block from file
*/
- private int[][] allSelectedDimensionBlocksIndexes;
+ private int[][] allSelectedDimensionColumnIndexRange;
/**
* will be used to read the measure block from file
*/
- private int[][] allSelectedMeasureBlocksIndexes;
+ private int[][] allSelectedMeasureIndexRange;
/**
* list of dimension present in the projection
@@ -118,16 +108,11 @@ public class BlockExecutionInfo {
/**
* dictionary column block indexes based on query
*/
- private int[] dictionaryColumnBlockIndex;
+ private int[] dictionaryColumnChunkIndex;
/**
* no dictionary column block indexes in based on the query order
*/
- private int[] noDictionaryBlockIndexes;
-
- /**
- * key generator used for generating the table block fixed length key
- */
- private KeyGenerator blockKeyGenerator;
+ private int[] noDictionaryColumnChunkIndexes;
/**
* each column value size
@@ -180,23 +165,23 @@ public class BlockExecutionInfo {
* list of dimension present in the current block. This will be
* different in case of restructured block
*/
- private QueryDimension[] queryDimensions;
+ private ProjectionDimension[] projectionDimensions;
/**
* list of dimension selected for in query
*/
- private QueryDimension[] actualQueryDimensions;
+ private ProjectionDimension[] actualQueryDimensions;
/**
* list of dimension present in the current block. This will be
* different in case of restructured block
*/
- private QueryMeasure[] queryMeasures;
+ private ProjectionMeasure[] projectionMeasures;
/**
* list of measure selected in query
*/
- private QueryMeasure[] actualQueryMeasures;
+ private ProjectionMeasure[] actualQueryMeasures;
/**
* variable to maintain dimension existence and default value info
@@ -214,11 +199,6 @@ public class BlockExecutionInfo {
private boolean isRestructuredBlock;
/**
- * absolute table identifier
- */
- private AbsoluteTableIdentifier absoluteTableIdentifier;
-
- /**
* delete delta file path
*/
private String[] deleteDeltaFilePath;
@@ -229,13 +209,6 @@ public class BlockExecutionInfo {
private boolean prefetchBlocklet = true;
private Map<String, DeleteDeltaVo> deletedRecordsMap;
- public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
- return absoluteTableIdentifier;
- }
-
- public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
- }
/**
* @param blockIndex the tableBlock to set
@@ -245,20 +218,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the isFixedKeyUpdateRequired
- */
- public boolean isFixedKeyUpdateRequired() {
- return isFixedKeyUpdateRequired;
- }
-
- /**
- * @param isFixedKeyUpdateRequired the isFixedKeyUpdateRequired to set
- */
- public void setFixedKeyUpdateRequired(boolean isFixedKeyUpdateRequired) {
- this.isFixedKeyUpdateRequired = isFixedKeyUpdateRequired;
- }
-
- /**
* @return the aggregatorInfos
*/
public MeasureInfo getMeasureInfo() {
@@ -301,59 +260,60 @@ public class BlockExecutionInfo {
}
/**
- * @return the totalNumberDimensionBlock
+ * @return the totalNumberDimensionToRead
*/
- public int getTotalNumberDimensionBlock() {
- return totalNumberDimensionBlock;
+ public int getTotalNumberDimensionToRead() {
+ return totalNumberDimensionToRead;
}
/**
- * @param totalNumberDimensionBlock the totalNumberDimensionBlock to set
+ * @param totalNumberDimensionToRead the totalNumberDimensionToRead to set
*/
- public void setTotalNumberDimensionBlock(int totalNumberDimensionBlock) {
- this.totalNumberDimensionBlock = totalNumberDimensionBlock;
+ public void setTotalNumberDimensionToRead(int totalNumberDimensionToRead) {
+ this.totalNumberDimensionToRead = totalNumberDimensionToRead;
}
/**
- * @return the totalNumberOfMeasureBlock
+ * @return the totalNumberOfMeasureToRead
*/
- public int getTotalNumberOfMeasureBlock() {
- return totalNumberOfMeasureBlock;
+ public int getTotalNumberOfMeasureToRead() {
+ return totalNumberOfMeasureToRead;
}
/**
- * @param totalNumberOfMeasureBlock the totalNumberOfMeasureBlock to set
+ * @param totalNumberOfMeasureToRead the totalNumberOfMeasureToRead to set
*/
- public void setTotalNumberOfMeasureBlock(int totalNumberOfMeasureBlock) {
- this.totalNumberOfMeasureBlock = totalNumberOfMeasureBlock;
+ public void setTotalNumberOfMeasureToRead(int totalNumberOfMeasureToRead) {
+ this.totalNumberOfMeasureToRead = totalNumberOfMeasureToRead;
}
/**
- * @return the allSelectedDimensionBlocksIndexes
+ * @return the allSelectedDimensionColumnIndexRange
*/
- public int[][] getAllSelectedDimensionBlocksIndexes() {
- return allSelectedDimensionBlocksIndexes;
+ public int[][] getAllSelectedDimensionColumnIndexRange() {
+ return allSelectedDimensionColumnIndexRange;
}
/**
- * @param allSelectedDimensionBlocksIndexes the allSelectedDimensionBlocksIndexes to set
+ * @param allSelectedDimensionColumnIndexRange the allSelectedDimensionColumnIndexRange to set
*/
- public void setAllSelectedDimensionBlocksIndexes(int[][] allSelectedDimensionBlocksIndexes) {
- this.allSelectedDimensionBlocksIndexes = allSelectedDimensionBlocksIndexes;
+ public void setAllSelectedDimensionColumnIndexRange(int[][] allSelectedDimensionColumnIndexRange)
+ {
+ this.allSelectedDimensionColumnIndexRange = allSelectedDimensionColumnIndexRange;
}
/**
- * @return the allSelectedMeasureBlocksIndexes
+ * @return the allSelectedMeasureIndexRange
*/
- public int[][] getAllSelectedMeasureBlocksIndexes() {
- return allSelectedMeasureBlocksIndexes;
+ public int[][] getAllSelectedMeasureIndexRange() {
+ return allSelectedMeasureIndexRange;
}
/**
- * @param allSelectedMeasureBlocksIndexes the allSelectedMeasureBlocksIndexes to set
+ * @param allSelectedMeasureIndexRange the allSelectedMeasureIndexRange to set
*/
- public void setAllSelectedMeasureBlocksIndexes(int[][] allSelectedMeasureBlocksIndexes) {
- this.allSelectedMeasureBlocksIndexes = allSelectedMeasureBlocksIndexes;
+ public void setAllSelectedMeasureIndexRange(int[][] allSelectedMeasureIndexRange) {
+ this.allSelectedMeasureIndexRange = allSelectedMeasureIndexRange;
}
/**
@@ -413,20 +373,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the tableBlockKeyGenerator
- */
- public KeyGenerator getBlockKeyGenerator() {
- return blockKeyGenerator;
- }
-
- /**
- * @param tableBlockKeyGenerator the tableBlockKeyGenerator to set
- */
- public void setBlockKeyGenerator(KeyGenerator tableBlockKeyGenerator) {
- this.blockKeyGenerator = tableBlockKeyGenerator;
- }
-
- /**
* @return the eachColumnValueSize
*/
public int[] getEachColumnValueSize() {
@@ -441,31 +387,31 @@ public class BlockExecutionInfo {
}
/**
- * @return the dictionaryColumnBlockIndex
+ * @return the dictionaryColumnChunkIndex
*/
- public int[] getDictionaryColumnBlockIndex() {
- return dictionaryColumnBlockIndex;
+ public int[] getDictionaryColumnChunkIndex() {
+ return dictionaryColumnChunkIndex;
}
/**
- * @param dictionaryColumnBlockIndex the dictionaryColumnBlockIndex to set
+ * @param dictionaryColumnChunkIndex the dictionaryColumnChunkIndex to set
*/
- public void setDictionaryColumnBlockIndex(int[] dictionaryColumnBlockIndex) {
- this.dictionaryColumnBlockIndex = dictionaryColumnBlockIndex;
+ public void setDictionaryColumnChunkIndex(int[] dictionaryColumnChunkIndex) {
+ this.dictionaryColumnChunkIndex = dictionaryColumnChunkIndex;
}
/**
- * @return the noDictionaryBlockIndexes
+ * @return the noDictionaryColumnChunkIndexes
*/
- public int[] getNoDictionaryBlockIndexes() {
- return noDictionaryBlockIndexes;
+ public int[] getNoDictionaryColumnChunkIndexes() {
+ return noDictionaryColumnChunkIndexes;
}
/**
- * @param noDictionaryBlockIndexes the noDictionaryBlockIndexes to set
+ * @param noDictionaryColumnChunkIndexes the noDictionaryColumnChunkIndexes to set
*/
- public void setNoDictionaryBlockIndexes(int[] noDictionaryBlockIndexes) {
- this.noDictionaryBlockIndexes = noDictionaryBlockIndexes;
+ public void setNoDictionaryColumnChunkIndexes(int[] noDictionaryColumnChunkIndexes) {
+ this.noDictionaryColumnChunkIndexes = noDictionaryColumnChunkIndexes;
}
/**
@@ -519,20 +465,20 @@ public class BlockExecutionInfo {
this.complexColumnParentBlockIndexes = complexColumnParentBlockIndexes;
}
- public QueryDimension[] getQueryDimensions() {
- return queryDimensions;
+ public ProjectionDimension[] getProjectionDimensions() {
+ return projectionDimensions;
}
- public void setQueryDimensions(QueryDimension[] queryDimensions) {
- this.queryDimensions = queryDimensions;
+ public void setProjectionDimensions(ProjectionDimension[] projectionDimensions) {
+ this.projectionDimensions = projectionDimensions;
}
- public QueryMeasure[] getQueryMeasures() {
- return queryMeasures;
+ public ProjectionMeasure[] getProjectionMeasures() {
+ return projectionMeasures;
}
- public void setQueryMeasures(QueryMeasure[] queryMeasures) {
- this.queryMeasures = queryMeasures;
+ public void setProjectionMeasures(ProjectionMeasure[] projectionMeasures) {
+ this.projectionMeasures = projectionMeasures;
}
/**
@@ -579,7 +525,8 @@ public class BlockExecutionInfo {
this.vectorBatchCollector = vectorBatchCollector;
}
- public String getBlockId() {
+ // Return file name and path, like Part0/Segment_0/part-0-0_batchno0-0-1517155583332.carbondata
+ public String getBlockIdString() {
return blockId;
}
@@ -603,19 +550,19 @@ public class BlockExecutionInfo {
this.dimensionInfo = dimensionInfo;
}
- public QueryDimension[] getActualQueryDimensions() {
+ public ProjectionDimension[] getActualQueryDimensions() {
return actualQueryDimensions;
}
- public void setActualQueryDimensions(QueryDimension[] actualQueryDimensions) {
+ public void setActualQueryDimensions(ProjectionDimension[] actualQueryDimensions) {
this.actualQueryDimensions = actualQueryDimensions;
}
- public QueryMeasure[] getActualQueryMeasures() {
+ public ProjectionMeasure[] getActualQueryMeasures() {
return actualQueryMeasures;
}
- public void setActualQueryMeasures(QueryMeasure[] actualQueryMeasures) {
+ public void setActualQueryMeasures(ProjectionMeasure[] actualQueryMeasures) {
this.actualQueryMeasures = actualQueryMeasures;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 8eb0a0d..00cdfe7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -60,9 +60,8 @@ import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.filter.TableProvider;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -83,7 +82,7 @@ public class QueryUtil {
* @param keyGenerator key generator
* @return masked key
*/
- public static int[] getMaskedByteRange(List<QueryDimension> queryDimensions,
+ public static int[] getMaskedByteRange(List<ProjectionDimension> queryDimensions,
KeyGenerator keyGenerator) {
Set<Integer> byteRangeSet = new TreeSet<Integer>();
int[] byteRange = null;
@@ -167,7 +166,7 @@ public class QueryUtil {
* @return max key for dimension
* @throws KeyGenException if any problem while generating the key
*/
- public static byte[] getMaxKeyBasedOnDimensions(List<QueryDimension> queryDimensions,
+ public static byte[] getMaxKeyBasedOnDimensions(List<ProjectionDimension> queryDimensions,
KeyGenerator generator) throws KeyGenException {
long[] max = new long[generator.getDimCount()];
Arrays.fill(max, 0L);
@@ -209,46 +208,46 @@ public class QueryUtil {
* on query dimension
*
* @param queryDimensions query dimension
- * @param dimensionOrdinalToBlockMapping mapping of dimension block in file to query dimension
+ * @param dimensionOrdinalToChunkMapping mapping of dimension block in file to query dimension
* @return block index of file
*/
- public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
- Map<Integer, Integer> dimensionOrdinalToBlockMapping,
+ public static int[] getDimensionChunkIndexes(List<ProjectionDimension> queryDimensions,
+ Map<Integer, Integer> dimensionOrdinalToChunkMapping,
List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions,
Set<Integer> allProjectionListDimensionIndexes) {
// using set as in row group columns will point to same block
- Set<Integer> dimensionBlockIndex = new HashSet<Integer>();
+ Set<Integer> dimensionChunkIndex = new HashSet<Integer>();
Set<Integer> filterDimensionOrdinal = getFilterDimensionOrdinal(filterDimensions);
- int blockIndex = 0;
+ int chunkIndex = 0;
for (int i = 0; i < queryDimensions.size(); i++) {
if (queryDimensions.get(i).getDimension().hasEncoding(Encoding.IMPLICIT)) {
continue;
}
Integer dimensionOrdinal = queryDimensions.get(i).getDimension().getOrdinal();
- allProjectionListDimensionIndexes.add(dimensionOrdinalToBlockMapping.get(dimensionOrdinal));
+ allProjectionListDimensionIndexes.add(dimensionOrdinalToChunkMapping.get(dimensionOrdinal));
if (queryDimensions.get(i).getDimension().getNumberOfChild() > 0) {
addChildrenBlockIndex(allProjectionListDimensionIndexes,
queryDimensions.get(i).getDimension());
}
if (!filterDimensionOrdinal.contains(dimensionOrdinal)) {
- blockIndex = dimensionOrdinalToBlockMapping.get(dimensionOrdinal);
- dimensionBlockIndex.add(blockIndex);
+ chunkIndex = dimensionOrdinalToChunkMapping.get(dimensionOrdinal);
+ dimensionChunkIndex.add(chunkIndex);
if (queryDimensions.get(i).getDimension().getNumberOfChild() > 0) {
- addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
+ addChildrenBlockIndex(dimensionChunkIndex, queryDimensions.get(i).getDimension());
}
}
}
for (int i = 0; i < customAggregationDimension.size(); i++) {
- blockIndex =
- dimensionOrdinalToBlockMapping.get(customAggregationDimension.get(i).getOrdinal());
+ chunkIndex =
+ dimensionOrdinalToChunkMapping.get(customAggregationDimension.get(i).getOrdinal());
// not adding the children dimension as dimension aggregation
// is not push down in case of complex dimension
- dimensionBlockIndex.add(blockIndex);
+ dimensionChunkIndex.add(chunkIndex);
}
int[] dimensionIndex = ArrayUtils
- .toPrimitive(dimensionBlockIndex.toArray(new Integer[dimensionBlockIndex.size()]));
+ .toPrimitive(dimensionChunkIndex.toArray(new Integer[dimensionChunkIndex.size()]));
Arrays.sort(dimensionIndex);
return dimensionIndex;
}
@@ -278,7 +277,7 @@ public class QueryUtil {
* @throws IOException
*/
public static Map<String, Dictionary> getDimensionDictionaryDetail(
- List<QueryDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions,
+ List<ProjectionDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions,
AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider)
throws IOException {
// to store complex dimension and its child id unique column id list, this is required as
@@ -433,23 +432,23 @@ public class QueryUtil {
* @param ordinalToBlockIndexMapping measure ordinal to block mapping
* @return block indexes
*/
- public static int[] getMeasureBlockIndexes(List<QueryMeasure> queryMeasures,
+ public static int[] getMeasureChunkIndexes(List<ProjectionMeasure> queryMeasures,
List<CarbonMeasure> expressionMeasure, Map<Integer, Integer> ordinalToBlockIndexMapping,
Set<CarbonMeasure> filterMeasures, List<Integer> allProjectionListMeasureIdexes) {
- Set<Integer> measureBlockIndex = new HashSet<Integer>();
+ Set<Integer> measureChunkIndex = new HashSet<Integer>();
Set<Integer> filterMeasureOrdinal = getFilterMeasureOrdinal(filterMeasures);
for (int i = 0; i < queryMeasures.size(); i++) {
Integer measureOrdinal = queryMeasures.get(i).getMeasure().getOrdinal();
allProjectionListMeasureIdexes.add(measureOrdinal);
if (!filterMeasureOrdinal.contains(measureOrdinal)) {
- measureBlockIndex.add(ordinalToBlockIndexMapping.get(measureOrdinal));
+ measureChunkIndex.add(ordinalToBlockIndexMapping.get(measureOrdinal));
}
}
for (int i = 0; i < expressionMeasure.size(); i++) {
- measureBlockIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()));
+ measureChunkIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()));
}
int[] measureIndexes =
- ArrayUtils.toPrimitive(measureBlockIndex.toArray(new Integer[measureBlockIndex.size()]));
+ ArrayUtils.toPrimitive(measureChunkIndex.toArray(new Integer[measureChunkIndex.size()]));
Arrays.sort(measureIndexes);
return measureIndexes;
}
@@ -462,8 +461,8 @@ public class QueryUtil {
* @param queryDimensions query dimension
* @return sort dimension indexes
*/
- public static byte[] getSortDimensionIndexes(List<QueryDimension> sortedDimensions,
- List<QueryDimension> queryDimensions) {
+ public static byte[] getSortDimensionIndexes(List<ProjectionDimension> sortedDimensions,
+ List<ProjectionDimension> queryDimensions) {
byte[] sortedDims = new byte[queryDimensions.size()];
int indexOf = 0;
for (int i = 0; i < sortedDims.length; i++) {
@@ -485,7 +484,7 @@ public class QueryUtil {
* @throws KeyGenException if problem while key generation
*/
public static Map<Integer, KeyStructureInfo> getColumnGroupKeyStructureInfo(
- List<QueryDimension> queryDimensions, SegmentProperties segmentProperties)
+ List<ProjectionDimension> queryDimensions, SegmentProperties segmentProperties)
throws KeyGenException {
Map<Integer, KeyStructureInfo> rowGroupToItsRSInfo = new HashMap<Integer, KeyStructureInfo>();
// get column group id and its ordinal mapping of column group
@@ -521,7 +520,7 @@ public class QueryUtil {
restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils
.toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()])));
rowGroupToItsRSInfo
- .put(segmentProperties.getDimensionOrdinalToBlockMapping().get(ordinal.get(0)),
+ .put(segmentProperties.getDimensionOrdinalToChunkMapping().get(ordinal.get(0)),
restructureInfos);
}
return rowGroupToItsRSInfo;
@@ -552,16 +551,16 @@ public class QueryUtil {
* stored in bit level
*/
private static Map<Integer, List<Integer>> getColumnGroupAndItsOrdinalMapping(
- List<QueryDimension> origdimensions) {
+ List<ProjectionDimension> origDimensions) {
- List<QueryDimension> dimensions = new ArrayList<QueryDimension>(origdimensions.size());
- dimensions.addAll(origdimensions);
+ List<ProjectionDimension> dimensions = new ArrayList<>(origDimensions.size());
+ dimensions.addAll(origDimensions);
/*
* sort based on column group id
*/
- Collections.sort(dimensions, new Comparator<QueryDimension>() {
+ Collections.sort(dimensions, new Comparator<ProjectionDimension>() {
- @Override public int compare(QueryDimension o1, QueryDimension o2) {
+ @Override public int compare(ProjectionDimension o1, ProjectionDimension o2) {
return Integer
.compare(o1.getDimension().columnGroupId(), o2.getDimension().columnGroupId());
}
@@ -636,70 +635,26 @@ public class QueryUtil {
* dictionary column. This is done for specific purpose so that in one
* iteration we will be able to fill both type dimension block indexes
*
- * @param queryDimensions dimension present in the query
- * @param columnOrdinalToBlockIndexMapping column ordinal to block index mapping
- * @param dictionaryDimensionBlockIndex list to store dictionary column block indexes
- * @param noDictionaryDimensionBlockIndex list to store no dictionary block indexes
+ * @param projectDimensions dimension present in the query
+ * @param columnOrdinalToChunkIndexMapping column ordinal to block index mapping
+ * @param dictionaryDimensionChunkIndex list to store dictionary column block indexes
+ * @param noDictionaryDimensionChunkIndex list to store no dictionary block indexes
*/
- public static void fillQueryDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
- Map<Integer, Integer> columnOrdinalToBlockIndexMapping,
- Set<Integer> dictionaryDimensionBlockIndex, List<Integer> noDictionaryDimensionBlockIndex) {
- for (QueryDimension queryDimension : queryDimensions) {
+ public static void fillQueryDimensionChunkIndexes(
+ List<ProjectionDimension> projectDimensions,
+ Map<Integer, Integer> columnOrdinalToChunkIndexMapping,
+ Set<Integer> dictionaryDimensionChunkIndex,
+ List<Integer> noDictionaryDimensionChunkIndex) {
+ for (ProjectionDimension queryDimension : projectDimensions) {
if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY)
&& queryDimension.getDimension().getNumberOfChild() == 0) {
- dictionaryDimensionBlockIndex
- .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
+ dictionaryDimensionChunkIndex
+ .add(columnOrdinalToChunkIndexMapping.get(queryDimension.getDimension().getOrdinal()));
} else if (
!CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.IMPLICIT)
&& queryDimension.getDimension().getNumberOfChild() == 0) {
- noDictionaryDimensionBlockIndex
- .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
- }
- }
- }
-
- /**
- * Below method will be used to resolve the query model
- * resolve will be setting the actual dimension and measure object
- * as from driver only column name will be passes to avoid the heavy object
- * serialization
- *
- * @param queryModel query model
- */
- public static void resolveQueryModel(QueryModel queryModel) {
- CarbonMetadata.getInstance().addCarbonTable(queryModel.getTable());
- // TODO need to load the table from table identifier
- CarbonTable carbonTable = queryModel.getTable();
- String tableName =
- queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName();
- // resolve query dimension
- for (QueryDimension queryDimension : queryModel.getQueryDimension()) {
- queryDimension
- .setDimension(carbonTable.getDimensionByName(tableName, queryDimension.getColumnName()));
- }
- // resolve query measure
- for (QueryMeasure queryMeasure : queryModel.getQueryMeasures()) {
- // in case of count start column name will be count * so
- // first need to check any measure is present or not and as if measure
- // if measure is present and if first measure is not a default
- // measure than add measure otherwise
- // than add first dimension as a measure
- //as currently if measure is not present then
- //we are adding default measure so first condition will
- //never come false but if in future we can remove so not removing first if check
- if (queryMeasure.getColumnName().equals("count(*)")) {
- if (carbonTable.getMeasureByTableName(tableName).size() > 0 && !carbonTable
- .getMeasureByTableName(tableName).get(0).getColName()
- .equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)) {
- queryMeasure.setMeasure(carbonTable.getMeasureByTableName(tableName).get(0));
- } else {
- CarbonMeasure dummyMeasure = new CarbonMeasure(
- carbonTable.getDimensionByTableName(tableName).get(0).getColumnSchema(), 0);
- queryMeasure.setMeasure(dummyMeasure);
- }
- } else {
- queryMeasure
- .setMeasure(carbonTable.getMeasureByName(tableName, queryMeasure.getColumnName()));
+ noDictionaryDimensionChunkIndex
+ .add(columnOrdinalToChunkIndexMapping.get(queryDimension.getDimension().getOrdinal()));
}
}
}
@@ -759,11 +714,11 @@ public class QueryUtil {
* @return complex dimension and query type
*/
public static Map<Integer, GenericQueryType> getComplexDimensionsMap(
- List<QueryDimension> queryDimensions, Map<Integer, Integer> dimensionToBlockIndexMap,
+ List<ProjectionDimension> queryDimensions, Map<Integer, Integer> dimensionToBlockIndexMap,
int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap,
Set<CarbonDimension> filterDimensions) {
Map<Integer, GenericQueryType> complexTypeMap = new HashMap<Integer, GenericQueryType>();
- for (QueryDimension dimension : queryDimensions) {
+ for (ProjectionDimension dimension : queryDimensions) {
CarbonDimension actualDimension = dimension.getDimension();
if (actualDimension.getNumberOfChild() == 0) {
continue;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index 288aea7..4874f1c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.DimensionInfo;
import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -61,11 +61,11 @@ public class RestructureUtil {
* @param tableComplexDimension
* @return list of query dimension which is present in the table block
*/
- public static List<QueryDimension> createDimensionInfoAndGetCurrentBlockQueryDimension(
- BlockExecutionInfo blockExecutionInfo, List<QueryDimension> queryDimensions,
+ public static List<ProjectionDimension> createDimensionInfoAndGetCurrentBlockQueryDimension(
+ BlockExecutionInfo blockExecutionInfo, List<ProjectionDimension> queryDimensions,
List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension) {
- List<QueryDimension> presentDimension =
- new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ List<ProjectionDimension> presentDimension =
+ new ArrayList<ProjectionDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
boolean[] isDimensionExists = new boolean[queryDimensions.size()];
Object[] defaultValues = new Object[queryDimensions.size()];
// create dimension information instance
@@ -74,22 +74,21 @@ public class RestructureUtil {
int newNoDictionaryColumnCount = 0;
// selecting only those dimension which is present in the query
int dimIndex = 0;
- for (QueryDimension queryDimension : queryDimensions) {
+ for (ProjectionDimension queryDimension : queryDimensions) {
if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
presentDimension.add(queryDimension);
isDimensionExists[dimIndex] = true;
} else {
for (CarbonDimension tableDimension : tableBlockDimensions) {
if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
- QueryDimension currentBlockDimension = new QueryDimension(tableDimension.getColName());
+ ProjectionDimension currentBlockDimension = new ProjectionDimension(tableDimension);
tableDimension.getColumnSchema()
.setPrecision(queryDimension.getDimension().getColumnSchema().getPrecision());
tableDimension.getColumnSchema()
.setScale(queryDimension.getDimension().getColumnSchema().getScale());
tableDimension.getColumnSchema()
.setDefaultValue(queryDimension.getDimension().getDefaultValue());
- currentBlockDimension.setDimension(tableDimension);
- currentBlockDimension.setQueryOrder(queryDimension.getQueryOrder());
+ currentBlockDimension.setOrdinal(queryDimension.getOrdinal());
presentDimension.add(currentBlockDimension);
isDimensionExists[dimIndex] = true;
break;
@@ -102,11 +101,10 @@ public class RestructureUtil {
}
for (CarbonDimension tableDimension : tableComplexDimension) {
if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
- QueryDimension currentBlockDimension = new QueryDimension(tableDimension.getColName());
+ ProjectionDimension currentBlockDimension = new ProjectionDimension(tableDimension);
// TODO: for complex dimension set scale and precision by traversing
// the child dimensions
- currentBlockDimension.setDimension(tableDimension);
- currentBlockDimension.setQueryOrder(queryDimension.getQueryOrder());
+ currentBlockDimension.setOrdinal(queryDimension.getOrdinal());
presentDimension.add(currentBlockDimension);
isDimensionExists[dimIndex] = true;
break;
@@ -242,39 +240,6 @@ public class RestructureUtil {
}
/**
- * Below method is to add dimension children for complex type dimension as
- * internally we are creating dimension column for each each complex
- * dimension so when complex query dimension request will come in the query,
- * we need to add its children as it is hidden from the user For example if
- * complex dimension is of Array of String[2] so we are storing 3 dimension
- * and when user will query for complex type i.e. array type we need to add
- * its children and then we will read respective block and create a tuple
- * based on all three dimension
- *
- * @param queryDimensions current query dimensions
- * @param tableBlockDimensions dimensions which is present in the table block
- * @return updated dimension(after adding complex type children)
- */
- public static List<CarbonDimension> addChildrenForComplexTypeDimension(
- List<CarbonDimension> queryDimensions, List<CarbonDimension> tableBlockDimensions) {
- List<CarbonDimension> updatedQueryDimension = new ArrayList<CarbonDimension>();
- int numberOfChildren = 0;
- for (CarbonDimension queryDimension : queryDimensions) {
- // if number of child is zero, then it is not a complex dimension
- // so directly add it query dimension
- if (queryDimension.getNumberOfChild() == 0) {
- updatedQueryDimension.add(queryDimension);
- }
- // if number of child is more than 1 then add all its children
- numberOfChildren = queryDimension.getOrdinal() + queryDimension.getNumberOfChild();
- for (int j = queryDimension.getOrdinal(); j < numberOfChildren; j++) {
- updatedQueryDimension.add(tableBlockDimensions.get(j));
- }
- }
- return updatedQueryDimension;
- }
-
- /**
* Method for computing measure default value based on the data type
*
* @param columnSchema
@@ -361,30 +326,29 @@ public class RestructureUtil {
* @param currentBlockMeasures current block measures
* @return measures present in the block
*/
- public static List<QueryMeasure> createMeasureInfoAndGetCurrentBlockQueryMeasures(
- BlockExecutionInfo blockExecutionInfo, List<QueryMeasure> queryMeasures,
+ public static List<ProjectionMeasure> createMeasureInfoAndGetCurrentBlockQueryMeasures(
+ BlockExecutionInfo blockExecutionInfo, List<ProjectionMeasure> queryMeasures,
List<CarbonMeasure> currentBlockMeasures) {
MeasureInfo measureInfo = new MeasureInfo();
- List<QueryMeasure> presentMeasure = new ArrayList<>(queryMeasures.size());
+ List<ProjectionMeasure> presentMeasure = new ArrayList<>(queryMeasures.size());
int numberOfMeasureInQuery = queryMeasures.size();
List<Integer> measureOrdinalList = new ArrayList<>(numberOfMeasureInQuery);
Object[] defaultValues = new Object[numberOfMeasureInQuery];
boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery];
int index = 0;
- for (QueryMeasure queryMeasure : queryMeasures) {
+ for (ProjectionMeasure queryMeasure : queryMeasures) {
// if query measure exists in current dimension measures
// then setting measure exists is true
// otherwise adding a default value of a measure
for (CarbonMeasure carbonMeasure : currentBlockMeasures) {
if (carbonMeasure.getColumnId().equals(queryMeasure.getMeasure().getColumnId())) {
- QueryMeasure currentBlockMeasure = new QueryMeasure(carbonMeasure.getColName());
+ ProjectionMeasure currentBlockMeasure = new ProjectionMeasure(carbonMeasure);
carbonMeasure.getColumnSchema().setDataType(queryMeasure.getMeasure().getDataType());
carbonMeasure.getColumnSchema().setPrecision(queryMeasure.getMeasure().getPrecision());
carbonMeasure.getColumnSchema().setScale(queryMeasure.getMeasure().getScale());
carbonMeasure.getColumnSchema()
.setDefaultValue(queryMeasure.getMeasure().getDefaultValue());
- currentBlockMeasure.setMeasure(carbonMeasure);
- currentBlockMeasure.setQueryOrder(queryMeasure.getQueryOrder());
+ currentBlockMeasure.setOrdinal(queryMeasure.getOrdinal());
presentMeasure.add(currentBlockMeasure);
measureOrdinalList.add(carbonMeasure.getOrdinal());
measureExistsInCurrentBlock[index] = true;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
index 981efb5..5cd2d34 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
@@ -70,10 +70,6 @@ public class ColumnExpression extends LeafExpression {
return columnName;
}
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
-
public boolean isDimension() {
return isDimension;
}
@@ -102,10 +98,6 @@ public class ColumnExpression extends LeafExpression {
return dataType;
}
- public void setDataType(DataType dataType) {
- this.dataType = dataType;
- }
-
@Override public ExpressionResult evaluate(RowIntf value) {
return new ExpressionResult(dataType, (null == value ? null : value.getVal(colIndex)));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
index dad3975..8b060e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
@@ -17,62 +17,29 @@
package org.apache.carbondata.core.scan.expression;
-import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
public class FilterModificationNode {
private Expression currentExp;
private Expression parentExp;
private ExpressionType expType;
- private ExpressionResult literalValue;
- private String columnName;
- public FilterModificationNode(Expression currentNode, Expression parentNode,
- ExpressionType expType, DataType dataType, Object literalVal, String colName) {
+ FilterModificationNode(Expression currentNode, Expression parentNode, ExpressionType expType) {
this.currentExp = currentNode;
this.parentExp = parentNode;
this.expType = expType;
- this.columnName = colName;
- this.literalValue = new ExpressionResult(dataType, literalVal);
}
public Expression getCurrentExp() {
return currentExp;
}
- public void setCurrentExp(Expression currentExp) {
- this.currentExp = currentExp;
- }
-
public Expression getParentExp() {
return parentExp;
}
- public void setParentExp(Expression parentExp) {
- this.parentExp = parentExp;
- }
-
public ExpressionType getExpType() {
return expType;
}
- public void setExpType(ExpressionType expType) {
- this.expType = expType;
- }
-
- public ExpressionResult getLiteralValue() {
- return literalValue;
- }
-
- public void setLiteralValue(ExpressionResult literalValue) {
- this.literalValue = literalValue;
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
index e7cc1c3..585bf60 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
@@ -48,7 +48,6 @@ public class RangeExpressionEvaluator {
LogServiceFactory.getLogService(RangeExpressionEvaluator.class.getName());
private Expression expr;
private Expression srcNode;
- private Expression srcParentNode;
private Expression tarNode;
private Expression tarParentNode;
@@ -64,15 +63,15 @@ public class RangeExpressionEvaluator {
this.expr = expr;
}
- public Expression getSrcNode() {
+ private Expression getSrcNode() {
return srcNode;
}
- public void setTarNode(Expression expr) {
+ private void setTarNode(Expression expr) {
this.tarNode = expr;
}
- public void setTarParentNode(Expression expr) {
+ private void setTarParentNode(Expression expr) {
this.tarParentNode = expr;
}
@@ -237,12 +236,10 @@ public class RangeExpressionEvaluator {
private void addFilterExpressionMap(Map<String, List<FilterModificationNode>> filterExpressionMap,
Expression currentNode, Expression parentNode) {
String colName = getColumnName(currentNode);
- DataType dataType = getLiteralDataType(currentNode);
- Object literalVal = getLiteralValue(currentNode);
ExpressionType expType = getExpressionType(currentNode);
FilterModificationNode filterExpression =
- new FilterModificationNode(currentNode, parentNode, expType, dataType, literalVal, colName);
+ new FilterModificationNode(currentNode, parentNode, expType);
if (null == filterExpressionMap.get(colName)) {
filterExpressionMap.put(colName, new ArrayList<FilterModificationNode>());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 5a1b7df..72ca1a4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -104,8 +104,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
*
*/
public List<DataRefNode> getFilterredBlocks(DataRefNode btreeNode,
- FilterResolverIntf filterResolver, AbstractIndex tableSegment,
- AbsoluteTableIdentifier tableIdentifier) {
+ FilterResolverIntf filterResolver, AbstractIndex tableSegment) {
// Need to get the current dimension tables
List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
// getting the start and end index key based on filter for hitting the
@@ -400,7 +399,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
case FALSE:
return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
case TRUE:
- return new TrueConditionalResolverImpl(expression, false, false, tableIdentifier);
+ return new TrueConditionalResolverImpl(expression, false, false);
case EQUALS:
currentCondExpression = (BinaryConditionalExpression) expression;
// check for implicit column in the expression
@@ -409,7 +408,6 @@ public class FilterExpressionProcessor implements FilterProcessor {
currentCondExpression.getColumnList().get(0).getCarbonColumn();
if (carbonColumn.hasEncoding(Encoding.IMPLICIT)) {
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier,
currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
}
@@ -435,7 +433,6 @@ public class FilterExpressionProcessor implements FilterProcessor {
tableIdentifier);
}
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier,
currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
// getting new dim index.
@@ -461,14 +458,12 @@ public class FilterExpressionProcessor implements FilterProcessor {
}
}
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier,
currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
break;
case RANGE:
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier, false);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true, false);
case NOT_EQUALS:
currentCondExpression = (BinaryConditionalExpression) expression;
column = currentCondExpression.getColumnList().get(0).getCarbonColumn();
@@ -491,8 +486,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, false,
tableIdentifier);
}
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier, true);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, true);
}
if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
@@ -515,11 +509,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
tableIdentifier);
}
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier, false);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, false);
}
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier, false);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, false);
}
break;
@@ -533,7 +525,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
.hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
.getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY))
|| (condExpression.getColumnList().get(0).getCarbonColumn().isMeasure())) {
- return new ConditionalFilterResolverImpl(expression, true, true, tableIdentifier,
+ return new ConditionalFilterResolverImpl(expression, true, true,
condExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
index dc392ba..dbb6eb7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
@@ -53,7 +53,7 @@ public interface FilterProcessor {
* @return list of DataRefNode.
*/
List<DataRefNode> getFilterredBlocks(DataRefNode dataRefNode, FilterResolverIntf filterResolver,
- AbstractIndex segmentIndexBuilder, AbsoluteTableIdentifier tableIdentifier);
+ AbstractIndex segmentIndexBuilder);
/**
* This API will get the map of required partitions.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 3268ca3..b8ffe0a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -49,7 +49,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -168,10 +168,8 @@ public final class FilterUtil {
segmentProperties);
case RANGE:
return new RangeValueFilterExecuterImpl(
- ((ConditionalFilterResolverImpl) filterExpressionResolverTree)
- .getDimColResolvedFilterInfo(),
- null, filterExpressionResolverTree.getFilterExpression(),
- ((ConditionalFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(),
+ filterExpressionResolverTree.getFilterExpression(),
((ConditionalFilterResolverImpl) filterExpressionResolverTree)
.getFilterRangeValues(segmentProperties), segmentProperties);
case TRUE:
@@ -545,10 +543,9 @@ public final class FilterUtil {
sortFilterModelMembers(columnExpression, evaluateResultList);
getDictionaryValue(evaluateResultList, forwardDictionary, surrogates);
filterInfo =
- getFilterValues(columnExpression, forwardDictionary, isIncludeFilter, null, surrogates);
+ getFilterValues(forwardDictionary, isIncludeFilter, surrogates);
if (filterInfo.isOptimized()) {
- return getDimColumnFilterInfoAfterApplyingCBO(columnExpression,
- forwardDictionary, filterInfo);
+ return getDimColumnFilterInfoAfterApplyingCBO(forwardDictionary, filterInfo);
}
} finally {
CarbonUtil.clearDictionaryCache(forwardDictionary);
@@ -560,22 +557,20 @@ public final class FilterUtil {
* Method will prepare the dimfilterinfo instance by resolving the filter
* expression value to its respective surrogates.
*
- * @param columnExpression
* @param forwardDictionary
* @param isIncludeFilter
* @param filterInfo
* @param surrogates
* @return
*/
- private static ColumnFilterInfo getFilterValues(ColumnExpression columnExpression,
- Dictionary forwardDictionary, boolean isIncludeFilter, ColumnFilterInfo filterInfo,
- List<Integer> surrogates) throws QueryExecutionException {
+ private static ColumnFilterInfo getFilterValues(Dictionary forwardDictionary,
+ boolean isIncludeFilter, List<Integer> surrogates) {
// Default value has to be added
if (surrogates.isEmpty()) {
surrogates.add(0);
}
boolean isExcludeFilterNeedsToApply = false;
- if (null == filterInfo && isIncludeFilter) {
+ if (isIncludeFilter) {
isExcludeFilterNeedsToApply =
isExcludeFilterNeedsToApply(forwardDictionary, surrogates.size());
}
@@ -587,17 +582,10 @@ public final class FilterUtil {
columnFilterInfo.setOptimized(true);
}
columnFilterInfo.setIncludeFilter(isIncludeFilter);
- if (null != filterInfo) {
- filterInfo.setIncludeFilter(isIncludeFilter);
- filterInfo.setOptimized(true);
- filterInfo.setExcludeFilterList(surrogates);
- return filterInfo;
+ if (!isIncludeFilter) {
+ columnFilterInfo.setExcludeFilterList(surrogates);
} else {
- if (!isIncludeFilter) {
- columnFilterInfo.setExcludeFilterList(surrogates);
- } else {
- columnFilterInfo.setFilterList(surrogates);
- }
+ columnFilterInfo.setFilterList(surrogates);
}
}
return columnFilterInfo;
@@ -613,8 +601,7 @@ public final class FilterUtil {
}
private static ColumnFilterInfo getDimColumnFilterInfoAfterApplyingCBO(
- ColumnExpression columnExpression, Dictionary forwardDictionary,
- ColumnFilterInfo filterInfo) throws FilterUnsupportedException, QueryExecutionException {
+ Dictionary forwardDictionary, ColumnFilterInfo filterInfo) throws FilterUnsupportedException {
List<Integer> excludeMemberSurrogates =
prepareExcludeFilterMembers(forwardDictionary, filterInfo.getFilterList());
filterInfo.setExcludeFilterList(excludeMemberSurrogates);
@@ -724,14 +711,12 @@ public final class FilterUtil {
prepareIncludeFilterMembers(expression, columnExpression, isIncludeFilter, forwardDictionary,
surrogates);
ColumnFilterInfo filterInfo =
- getFilterValues(columnExpression, forwardDictionary, isIncludeFilter, null, surrogates);
+ getFilterValues(forwardDictionary, isIncludeFilter, surrogates);
if (filterInfo.isOptimized()) {
- return getDimColumnFilterInfoAfterApplyingCBO(columnExpression, forwardDictionary,
+ return getDimColumnFilterInfoAfterApplyingCBO(forwardDictionary,
filterInfo);
}
return filterInfo;
- } catch (QueryExecutionException e) {
- throw new FilterUnsupportedException(e.getMessage());
} finally {
CarbonUtil.clearDictionaryCache(forwardDictionary);
}
@@ -755,21 +740,16 @@ public final class FilterUtil {
* expression value to its respective surrogates in the scenario of restructure.
*
* @param expression
- * @param columnExpression
* @param defaultValues
* @param defaultSurrogate
* @return
* @throws FilterUnsupportedException
*/
- public static ColumnFilterInfo getFilterListForRS(Expression expression,
- ColumnExpression columnExpression, String defaultValues, int defaultSurrogate)
- throws FilterUnsupportedException {
+ public static ColumnFilterInfo getFilterListForRS(Expression expression, String defaultValues,
+ int defaultSurrogate) throws FilterUnsupportedException {
List<Integer> filterValuesList = new ArrayList<Integer>(20);
ColumnFilterInfo columnFilterInfo = null;
- // List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
List<String> evaluateResultListFinal = new ArrayList<String>(20);
- // KeyGenerator keyGenerator =
- // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
try {
List<ExpressionResult> evaluateResultList = expression.evaluate(null).getList();
for (ExpressionResult result : evaluateResultList) {
@@ -815,8 +795,6 @@ public final class FilterUtil {
List<String> evaluateResultListFinal = new ArrayList<String>(20);
ColumnFilterInfo columnFilterInfo = null;
- // KeyGenerator keyGenerator =
- // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
try {
RowIntf row = new RowImpl();
if (defaultValues.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
@@ -1769,14 +1747,14 @@ public final class FilterUtil {
* This method will compare the selected data against null values and
* flip the bitSet if any null value is found
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param bitSet
*/
- public static void removeNullValues(DimensionColumnDataChunk dimensionColumnDataChunk,
+ public static void removeNullValues(DimensionColumnPage dimensionColumnPage,
BitSet bitSet, byte[] defaultValue) {
if (!bitSet.isEmpty()) {
for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
- if (dimensionColumnDataChunk.compareTo(i, defaultValue) == 0) {
+ if (dimensionColumnPage.compareTo(i, defaultValue) == 0) {
bitSet.flip(i);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
index 3742e7e..214bd9d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.spark.sql.types.DataType;
@@ -45,7 +45,7 @@ public interface GenericQueryType {
DataType getSchemaType();
- void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) throws IOException;
+ void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder) throws IOException;
Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
index f882162..e309451 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
@@ -21,7 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class AndFilterExecuterImpl implements FilterExecuter, ImplicitColumnFilterExecutor {
@@ -35,18 +35,18 @@ public class AndFilterExecuterImpl implements FilterExecuter, ImplicitColumnFilt
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, useBitsetPipeLine);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ BitSetGroup leftFilters = leftExecuter.applyFilter(rawBlockletColumnChunks, useBitsetPipeLine);
if (leftFilters.isEmpty()) {
return leftFilters;
}
- BitSetGroup rightFilter = rightExecuter.applyFilter(blockChunkHolder, useBitsetPipeLine);
+ BitSetGroup rightFilter = rightExecuter.applyFilter(rawBlockletColumnChunks, useBitsetPipeLine);
if (rightFilter.isEmpty()) {
return rightFilter;
}
leftFilters.and(rightFilter);
- blockChunkHolder.setBitSetGroup(leftFilters);
+ rawBlockletColumnChunks.setBitSetGroup(leftFilters);
return leftFilters;
}
@@ -69,9 +69,10 @@ public class AndFilterExecuterImpl implements FilterExecuter, ImplicitColumnFilt
return leftFilters;
}
- @Override public void readBlocks(BlocksChunkHolder blocksChunkHolder) throws IOException {
- leftExecuter.readBlocks(blocksChunkHolder);
- rightExecuter.readBlocks(blocksChunkHolder);
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ leftExecuter.readColumnChunks(rawBlockletColumnChunks);
+ rightExecuter.readColumnChunks(rawBlockletColumnChunks);
}
@Override
@@ -93,8 +94,7 @@ public class AndFilterExecuterImpl implements FilterExecuter, ImplicitColumnFilt
rightFilter = ((ImplicitColumnFilterExecutor) rightExecuter)
.isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath);
} else {
- rightFilter = rightExecuter
- .isScanRequired(maxValue, minValue);
+ rightFilter = rightExecuter.isScanRequired(maxValue, minValue);
}
if (rightFilter.isEmpty()) {
return rightFilter;
[16/50] [abbrv] carbondata git commit: [CARBONDATA-2097] Restriction
added to partition table on alter command (add,
rename on partition table and drop partition on preaggregate table)
Posted by ra...@apache.org.
[CARBONDATA-2097] Restriction added to partition table on alter command (add,rename on partition table and drop partition on preaggregate table)
Restriction added to partition table on alter command (add and rename on partion table and drop partition on preaggregate table)
This closes #1885
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/033870da
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/033870da
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/033870da
Branch: refs/heads/fgdatamap
Commit: 033870dab137ba99f5273c934741e96884b3247d
Parents: 099a047
Author: kushalsaha <ku...@gmail.com>
Authored: Tue Jan 30 16:38:47 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Wed Jan 31 14:46:54 2018 +0530
----------------------------------------------------------------------
.../StandardPartitionTableQueryTestCase.scala | 42 ++++++++++++++++++++
...rbonAlterTableDropHivePartitionCommand.scala | 7 +++-
.../sql/execution/strategy/DDLStrategy.scala | 21 ++++++++++
3 files changed, 69 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/033870da/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
index d1ef94c..b1fc0a7 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
@@ -248,6 +248,46 @@ test("Creation of partition table should fail if the colname in table schema and
}
}
+ test("Renaming a partition table should fail"){
+ sql("drop table if exists partitionTable")
+ sql(
+ """create table partitionTable (id int,name String) partitioned by(email string) stored by 'carbondata'
+ """.stripMargin)
+ sql("insert into partitionTable select 1,'huawei','abc'")
+ checkAnswer(sql("show partitions partitionTable"), Seq(Row("email=abc")))
+ intercept[Exception]{
+ sql("alter table partitionTable PARTITION (email='abc') rename to PARTITION (email='def)")
+ }
+ }
+
+ test("add partition based on location on partition table should fail"){
+ sql("drop table if exists partitionTable")
+ sql(
+ """create table partitionTable (id int,name String) partitioned by(email string) stored by 'carbondata'
+ """.stripMargin)
+ sql("insert into partitionTable select 1,'huawei','abc'")
+ checkAnswer(sql("show partitions partitionTable"), Seq(Row("email=abc")))
+ intercept[Exception]{
+ sql("alter table partitionTable add partition (email='def') location 'abc/part1'")
+ }
+ }
+
+ test("drop partition on preAggregate table should fail"){
+ sql("drop table if exists partitionTable")
+ sql("drop datamap if exists preaggTable on table partitionTable")
+ sql("create table partitionTable (id int,city string,age int) partitioned by(name string) stored by 'carbondata'".stripMargin)
+ sql(
+ s"""create datamap preaggTable on table partitionTable using 'preaggregate' as select id,sum(age) from partitionTable group by id"""
+ .stripMargin)
+ sql("insert into partitionTable select 1,'Bangalore',30,'John'")
+ sql("insert into partitionTable select 2,'Chennai',20,'Huawei'")
+ checkAnswer(sql("show partitions partitionTable"), Seq(Row("name=John"),Row("name=Huawei")))
+ intercept[Exception]{
+ sql("alter table partitionTable drop PARTITION(name='John')")
+ }
+ }
+
+
private def verifyPartitionInfo(frame: DataFrame, partitionNames: Seq[String]) = {
val plan = frame.queryExecution.sparkPlan
val scanRDD = plan collect {
@@ -277,6 +317,8 @@ test("Creation of partition table should fail if the colname in table schema and
sql("drop table if exists badrecordsignore")
sql("drop table if exists badrecordsPartitionintnull")
sql("drop table if exists badrecordsPartitionintnullalt")
+ sql("drop table if exists partitionTable")
+ sql("drop datamap if exists preaggTable on table partitionTable")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/033870da/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
index c3509a3..0158a32 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
@@ -21,7 +21,7 @@ import java.util
import scala.collection.JavaConverters._
-import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.{AnalysisException, CarbonEnv, Row, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, AlterTableDropPartitionCommand, AtomicRunnableCommand}
@@ -34,6 +34,7 @@ import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.PartitionMapFileStore
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
+import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.spark.rdd.{CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD}
/**
@@ -61,6 +62,10 @@ case class CarbonAlterTableDropHivePartitionCommand(
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ if (CarbonUtil.hasAggregationDataMap(table)) {
+ throw new AnalysisException(
+ "Partition can not be dropped as it is mapped to Pre Aggregate table")
+ }
if (table.isHivePartitionTable) {
try {
specs.flatMap(f => sparkSession.sessionState.catalog.listPartitions(tableName, Some(f)))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/033870da/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index db8c6a2..b174b94 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -254,6 +254,27 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
ExecutedCommandExec(
CarbonAlterTableUnsetCommand(tableName, propKeys, ifExists, isView)) :: Nil
}
+ case rename@AlterTableRenamePartitionCommand(tableName, oldPartition, newPartition) =>
+ val dbOption = tableName.database.map(_.toLowerCase)
+ val tableIdentifier = TableIdentifier(tableName.table.toLowerCase(), dbOption)
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .tableExists(tableIdentifier)(sparkSession)
+ if (isCarbonTable) {
+ throw new UnsupportedOperationException("Renaming partition on table is not supported")
+ } else {
+ ExecutedCommandExec(rename) :: Nil
+ }
+ case addPartition@AlterTableAddPartitionCommand(tableName, partitionSpecsAndLocs, _) =>
+ val dbOption = tableName.database.map(_.toLowerCase)
+ val tableIdentifier = TableIdentifier(tableName.table.toLowerCase(), dbOption)
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .tableExists(tableIdentifier)(sparkSession)
+ if (isCarbonTable && partitionSpecsAndLocs.exists(_._2.isDefined)) {
+ throw new UnsupportedOperationException(
+ "add partition with location is not supported")
+ } else {
+ ExecutedCommandExec(addPartition) :: Nil
+ }
case RefreshTable(tableIdentifier) =>
RefreshCarbonTableCommand(tableIdentifier.database,
tableIdentifier.table).run(sparkSession)
[24/50] [abbrv] carbondata git commit: [CARBONDATA-2089]SQL exception
is masked due to assert(false) inside try catch and exception block always
asserting true
Posted by ra...@apache.org.
[CARBONDATA-2089]SQL exception is masked due to assert(false) inside try catch and exception block always asserting true
Correct all SDV testcase to use intercept exception
This closes #1871
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3dff273b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3dff273b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3dff273b
Branch: refs/heads/fgdatamap
Commit: 3dff273b4f1308fa76a91f6f22bb40eb2d2d9553
Parents: b2139ca
Author: Raghunandan S <ca...@gmail.com>
Authored: Sat Jan 27 20:49:47 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 19:28:09 2018 +0800
----------------------------------------------------------------------
.../sdv/generated/AlterTableTestCase.scala | 250 ++++++---------
.../sdv/generated/BatchSortLoad1TestCase.scala | 39 +--
.../sdv/generated/BatchSortLoad2TestCase.scala | 32 +-
.../sdv/generated/BatchSortQueryTestCase.scala | 290 +++--------------
.../sdv/generated/BucketingTestCase.scala | 12 +-
.../sdv/generated/ColumndictTestCase.scala | 60 +---
.../sdv/generated/DataLoadingIUDTestCase.scala | 318 ++++++++-----------
.../sdv/generated/DataLoadingTestCase.scala | 7 +-
.../sdv/generated/InvertedindexTestCase.scala | 14 +-
.../sdv/generated/OffheapQuery1TestCase.scala | 287 +++--------------
.../sdv/generated/OffheapQuery2TestCase.scala | 286 +++--------------
.../sdv/generated/OffheapSort1TestCase.scala | 10 +-
.../sdv/generated/OffheapSort2TestCase.scala | 10 +-
.../sdv/generated/PartitionTestCase.scala | 71 ++---
.../sdv/generated/SinglepassTestCase.scala | 76 ++---
15 files changed, 423 insertions(+), 1339 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala
index b1a0f34..8899f5c 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala
@@ -120,141 +120,107 @@ class AlterTableTestCase extends QueryTest with BeforeAndAfterAll {
//Check alter table when the altered name is already present in the database
test("RenameTable_001_08", Include) {
- try {
- sql(s"""create table test1 (name string, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx',1""").collect
- sql(s"""create table test2 (name string, id int) stored by 'carbondata'""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (name string, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx',1""").collect
+ sql(s"""create table test2 (name string, id int) stored by 'carbondata'""").collect
sql(s"""alter table test1 RENAME TO test2""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
- sql(s"""drop table if exists test2""").collect
+
+ sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test2""").collect
}
//Check alter table when the altered name is given multiple times
test("RenameTable_001_09", Include) {
- try {
- sql(s"""create table test1 (name string, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx',1""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (name string, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx',1""").collect
sql(s"""alter table test1 RENAME TO test2 test3""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
//Check delete column for dimension column
test("DeleteCol_001_01", Include) {
- try {
- sql(s"""create table test1 (name string, id int) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='id') """).collect
- sql(s"""insert into test1 select 'xx',1""").collect
- sql(s"""alter table test1 drop columns (name)""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (name string, id int) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='id') """).collect
+ sql(s"""insert into test1 select 'xx',1""").collect
+ sql(s"""alter table test1 drop columns (name)""").collect
sql(s"""select name from test1""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
//Check delete column for measure column
test("DeleteCol_001_02", Include) {
- try {
- sql(s"""create table test1 (name string, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx',1""").collect
- sql(s"""alter table test1 drop columns (id)""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (name string, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx',1""").collect
+ sql(s"""alter table test1 drop columns (id)""").collect
sql(s"""select id from test1""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
//Check delete column for measure and dimension column
test("DeleteCol_001_03", Include) {
- try {
- sql(s"""create table test1 (name string, country string, upd_time timestamp, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx','yy',current_timestamp,1""").collect
- sql(s"""alter table test1 drop columns (id,name)""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (name string, country string, upd_time timestamp, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx','yy',current_timestamp,1""").collect
+ sql(s"""alter table test1 drop columns (id,name)""").collect
sql(s"""select id,name from test1""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
//Check delete column for multiple column
test("DeleteCol_001_04", Include) {
- try {
- sql(s"""create table test1 (name string, country string, upd_time timestamp, id int) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='id')""").collect
- sql(s"""insert into test1 select 'xx','yy',current_timestamp,1""").collect
- sql(s"""alter table test1 drop columns (name, upd_time)""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (name string, country string, upd_time timestamp, id int) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='id')""").collect
+ sql(s"""insert into test1 select 'xx','yy',current_timestamp,1""").collect
+ sql(s"""alter table test1 drop columns (name, upd_time)""").collect
sql(s"""select name, upd_time from test1""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
//Check delete column for all columns
test("DeleteCol_001_05", Include) {
- try {
- sql(s"""create table test1 (name string, country string, upd_time timestamp, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx','yy',current_timestamp,1""").collect
- sql(s"""alter table test1 drop columns (name, upd_time, country,id)""").collect
- assert(false)
- } catch {
- case _ => assert(true)
- }
- sql(s"""drop table if exists test1""").collect
+ sql(s"""create table test1 (name string, country string, upd_time timestamp, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx','yy',current_timestamp,1""").collect
+ sql(s"""alter table test1 drop columns (name, upd_time, country,id)""").collect
+ sql(s"""drop table if exists test1""").collect
}
//Check delete column for include dictionary column
test("DeleteCol_001_06", Include) {
- try {
- sql(s"""create table test1 (name string, id int) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='id')""").collect
- sql(s"""insert into test1 select 'xx',1""").collect
- sql(s"""alter table test1 drop columns (id)""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (name string, id int) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='id')""").collect
+ sql(s"""insert into test1 select 'xx',1""").collect
+ sql(s"""alter table test1 drop columns (id)""").collect
sql(s"""select id from test1""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
//Check delete column for timestamp column
test("DeleteCol_001_08", Include) {
- try {
- sql(s"""create table test1 (name string, country string, upd_time timestamp, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx','yy',current_timestamp,1""").collect
- sql(s"""alter table test1 drop columns (upd_time)""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (name string, country string, upd_time timestamp, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx','yy',current_timestamp,1""").collect
+ sql(s"""alter table test1 drop columns (upd_time)""").collect
sql(s"""select upd_time from test1""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
@@ -272,17 +238,13 @@ class AlterTableTestCase extends QueryTest with BeforeAndAfterAll {
//Check the drop of added column will remove the column from table
test("DeleteCol_001_09_2", Include) {
- try {
+ intercept[Exception] {
sql(s"""create table test1 (name string, country string, upd_time timestamp, id int) stored by 'carbondata'""").collect
sql(s"""insert into test1 select 'xx','yy',current_timestamp,1""").collect
sql(s"""alter table test1 add columns (name2 string)""").collect
sql(s"""insert into test1 select 'xx','yy',current_timestamp,1,'abc'""").collect
sql(s"""alter table test1 drop columns (name2)""").collect
sql(s"""select count(id) from test1 where name2 = 'abc'""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists test1""").collect
}
@@ -451,16 +413,13 @@ class AlterTableTestCase extends QueryTest with BeforeAndAfterAll {
//Check add column with option default value is given for an existing column
test("AddColumn_001_14", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists test1""").collect
sql(s"""create table test1 (name string) stored by 'carbondata'""").collect
sql(s"""insert into test1 select 'xx'""").collect
sql(s"""ALTER TABLE test1 ADD COLUMNS (Id int) TBLPROPERTIES('DICTIONARY_INCLUDE'='id','default.value.name'='yy')""").collect
- assert(false)
- sql(s"""drop table if exists test1""").collect
- } catch {
- case _ => assert(true)
}
+ sql(s"""drop table if exists test1""").collect
}
@@ -489,17 +448,14 @@ class AlterTableTestCase extends QueryTest with BeforeAndAfterAll {
//check drop table after table rename using old name
test("DropTable_001_02", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists test1""").collect
- sql(s"""create table test1 (name string, price decimal(3,2)) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx',1.2""").collect
- sql(s"""alter table test1 rename to test2""").collect
+ sql(s"""create table test1 (name string, price decimal(3,2)) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx',1.2""").collect
+ sql(s"""alter table test1 rename to test2""").collect
sql(s"""drop table test1""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test2""").collect
+ sql(s"""drop table if exists test2""").collect
}
@@ -734,15 +690,12 @@ class AlterTableTestCase extends QueryTest with BeforeAndAfterAll {
//Check show segments on old table After altering the Table name.
test("Showsegme_001_01", Include) {
- try {
- sql(s"""create table test1 (country string, id int) stored by 'carbondata'""").collect
- sql(s"""alter table test1 rename to test2""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (country string, id int) stored by 'carbondata'""").collect
+ sql(s"""alter table test1 rename to test2""").collect
sql(s"""show segments for table test1""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test2""").collect
+ sql(s"""drop table if exists test2""").collect
}
@@ -828,65 +781,53 @@ class AlterTableTestCase extends QueryTest with BeforeAndAfterAll {
//Check vertical compaction when all segments are created before drop column, check dropped column is not used in the compation
test("Compaction_001_06", Include) {
- try {
- sql(s"""drop table if exists test1""").collect
- sql(s"""drop table if exists test2""").collect
- sql(s"""create table test1(name string, country string, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx','china',1""").collect
- sql(s"""insert into test1 select 'xe','china',2""").collect
- sql(s"""insert into test1 select 'xe','china',3""").collect
- sql(s"""alter table test1 drop columns (country)""").collect
- sql(s"""alter table test1 compact 'minor'""").collect
+ intercept[Exception] {
+ sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test2""").collect
+ sql(s"""create table test1(name string, country string, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx','china',1""").collect
+ sql(s"""insert into test1 select 'xe','china',2""").collect
+ sql(s"""insert into test1 select 'xe','china',3""").collect
+ sql(s"""alter table test1 drop columns (country)""").collect
+ sql(s"""alter table test1 compact 'minor'""").collect
sql(s"""select country from test1 where country='china'""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
//Check vertical compaction when some of the segments are created before drop column, check dropped column is not used in the compation
test("Compaction_001_07", Include) {
- try {
- sql(s"""drop table if exists test1""").collect
- sql(s"""drop table if exists test2""").collect
- sql(s"""create table test1(name string, country string, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx','china',1""").collect
- sql(s"""insert into test1 select 'xe','china',2""").collect
- sql(s"""alter table test1 drop columns (country)""").collect
- sql(s"""insert into test1 select 'xe',3""").collect
- sql(s"""alter table test1 compact 'minor'""").collect
+ intercept[Exception] {
+ sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test2""").collect
+ sql(s"""create table test1(name string, country string, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx','china',1""").collect
+ sql(s"""insert into test1 select 'xe','china',2""").collect
+ sql(s"""alter table test1 drop columns (country)""").collect
+ sql(s"""insert into test1 select 'xe',3""").collect
+ sql(s"""alter table test1 compact 'minor'""").collect
sql(s"""select country from test1 where country='china'""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
//Check vertical compaction for multiple drop column, check dropped column is not used in the compation
test("Compaction_001_08", Include) {
- try {
- sql(s"""drop table if exists test1""").collect
- sql(s"""drop table if exists test2""").collect
- sql(s"""create table test1(name string, country string, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx','china',1""").collect
- sql(s"""alter table test1 drop columns (country)""").collect
- sql(s"""insert into test1 select 'xe',3""").collect
- sql(s"""alter table test1 drop columns (id)""").collect
- sql(s"""insert into test1 select 'xe'""").collect
- sql(s"""alter table test1 compact 'minor'""").collect
+ intercept[Exception] {
+ sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test2""").collect
+ sql(s"""create table test1(name string, country string, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx','china',1""").collect
+ sql(s"""alter table test1 drop columns (country)""").collect
+ sql(s"""insert into test1 select 'xe',3""").collect
+ sql(s"""alter table test1 drop columns (id)""").collect
+ sql(s"""insert into test1 select 'xe'""").collect
+ sql(s"""alter table test1 compact 'minor'""").collect
sql(s"""select country from test1 where id=1""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test1""").collect
+ sql(s"""drop table if exists test1""").collect
}
@@ -989,17 +930,14 @@ class AlterTableTestCase extends QueryTest with BeforeAndAfterAll {
//Check delete segment is not allowed on old table name when table name is altered
test("DeleteSeg_001_01", Include) {
- try {
- sql(s"""create table test1 (name string, id int) stored by 'carbondata'""").collect
- sql(s"""insert into test1 select 'xx',1""").collect
- sql(s"""insert into test1 select 'xx',12""").collect
- sql(s"""alter table test1 rename to test2""").collect
+ intercept[Exception] {
+ sql(s"""create table test1 (name string, id int) stored by 'carbondata'""").collect
+ sql(s"""insert into test1 select 'xx',1""").collect
+ sql(s"""insert into test1 select 'xx',12""").collect
+ sql(s"""alter table test1 rename to test2""").collect
sql(s"""delete from table test1 where segment.id in (0)""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table if exists test2""").collect
+ sql(s"""drop table if exists test2""").collect
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala
index 9eb5dec..d301218 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad1TestCase.scala
@@ -68,27 +68,21 @@ class BatchSortLoad1TestCase extends QueryTest with BeforeAndAfterAll {
//To load data after setting sort scope and sort size in carbon property file without folder path in load
test("Batch_sort_Loading_001-01-01-01_001-TC_004", Include) {
- try {
- sql(s"""CREATE TABLE uniqdata13 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
+ intercept[Exception] {
+ sql(s"""CREATE TABLE uniqdata13 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA into table uniqdata13 OPTIONS('DELIMITER'=',' , 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table uniqdata13""").collect
+ sql(s"""drop table uniqdata13""").collect
}
//To load data after setting sort scope and sort size in carbon property file without table_name in load
test("Batch_sort_Loading_001-01-01-01_001-TC_005", Include) {
- try {
- sql(s"""CREATE TABLE uniqdata14 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
+ intercept[Exception] {
+ sql(s"""CREATE TABLE uniqdata14 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' into table OPTIONS('DELIMITER'=',' , 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table uniqdata14""").collect
+ sql(s"""drop table uniqdata14""").collect
}
@@ -232,14 +226,11 @@ class BatchSortLoad1TestCase extends QueryTest with BeforeAndAfterAll {
//To load data after setting sort scope and sort size in carbon property file with ALL_DICTIONARY_PATH
test("Batch_sort_Loading_001-01-01-01_001-TC_019", Include) {
sql(s"""drop table if exists t3""").collect
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/batchsort/data.csv' into table t3 options('ALL_DICTIONARY_PATH'='resourcesPath/Data/batchsort/data.dictionary')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table t3""").collect
+ sql(s"""drop table t3""").collect
}
@@ -260,22 +251,16 @@ class BatchSortLoad1TestCase extends QueryTest with BeforeAndAfterAll {
//To check sort_scope option with a wrong value
test("Batch_sort_Loading_001-01-01-01_001-TC_023", Include) {
- try {
- sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='ABCXYZ')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
+ intercept[Exception] {
+ sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='ABCXYZ')""").collect
}
}
//To check sort_scope option with null value
test("Batch_sort_Loading_001-01-01-01_001-TC_024", Include) {
- try {
- sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='null')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
+ intercept[Exception] {
+ sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='null')""").collect
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad2TestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad2TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad2TestCase.scala
index 5fa6594..d3ff6aa 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad2TestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortLoad2TestCase.scala
@@ -69,27 +69,21 @@ class BatchSortLoad2TestCase extends QueryTest with BeforeAndAfterAll {
//To load data after setting only sort scope in carbon property file without folder path in load
test("Batch_sort_Loading_001-01-01-01_001-TC_030", Include) {
- try {
- sql(s"""CREATE TABLE uniqdata13 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
+ intercept[Exception] {
+ sql(s"""CREATE TABLE uniqdata13 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA into table uniqdata13 OPTIONS('DELIMITER'=',' , 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table uniqdata13""").collect
+ sql(s"""drop table uniqdata13""").collect
}
//To load data after setting only sort scope in carbon property file without table_name in load
test("Batch_sort_Loading_001-01-01-01_001-TC_031", Include) {
- try {
- sql(s"""CREATE TABLE uniqdata14 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
+ intercept[Exception] {
+ sql(s"""CREATE TABLE uniqdata14 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' into table OPTIONS('DELIMITER'=',' , 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table uniqdata14""").collect
+ sql(s"""drop table uniqdata14""").collect
}
@@ -255,12 +249,9 @@ class BatchSortLoad2TestCase extends QueryTest with BeforeAndAfterAll {
//To check sort_scope option with a wrong value
test("Batch_sort_Loading_001-01-01-01_001-TC_049", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' , 'SORT_SCOPE'='ABCXYZ',‘SINGLE_PASS’=’true’,'QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table uniqdata20a""").collect
}
@@ -268,14 +259,11 @@ class BatchSortLoad2TestCase extends QueryTest with BeforeAndAfterAll {
//To check sort_scope option with null value
test("Batch_sort_Loading_001-01-01-01_001-TC_050", Include) {
- try {
- sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect
+ intercept[Exception] {
+ sql(s"""CREATE TABLE uniqdata20a (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/7000_UniqData.csv' into table uniqdata20a OPTIONS('DELIMITER'=',' , 'SORT_SCOPE'='null',‘SINGLE_PASS’=’true’,'QUOTECHAR'='"','COMMENTCHAR'='#','MULTILINE'='true','ESCAPECHAR'='\','BAD_RECORDS_ACTION'='REDIRECT','BAD_RECORDS_LOGGER_ENABLE'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table uniqdata20a""").collect
+ sql(s"""drop table uniqdata20a""").collect
}
val prop = CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortQueryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortQueryTestCase.scala
index cdebf51..11b060a 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortQueryTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BatchSortQueryTestCase.scala
@@ -44,15 +44,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check select query with limit as string
test("Batch_sort_Querying_001-01-01-01_001-TC_002", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 limit """"").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -110,57 +104,33 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check where clause with OR and no operand
test("Batch_sort_Querying_001-01-01-01_001-TC_009", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id > 1 OR """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check OR clause with LHS and RHS having no arguments
test("Batch_sort_Querying_001-01-01-01_001-TC_010", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where OR """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check OR clause with LHS having no arguments
test("Batch_sort_Querying_001-01-01-01_001-TC_011", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where OR cust_id > "1"""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check incorrect query
test("Batch_sort_Querying_001-01-01-01_001-TC_013", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id > 0 OR name """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -229,15 +199,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check select count and distinct query execution
test("Batch_sort_Querying_001-01-01-01_001-TC_021", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select count(cust_id),distinct(cust_name) from uniqdataquery1""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -279,15 +243,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check query execution with IN operator without paranthesis
test("Batch_sort_Querying_001-01-01-01_001-TC_027", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id IN 9000,9005""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -302,15 +260,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check query execution with IN operator with out specifying any field.
test("Batch_sort_Querying_001-01-01-01_001-TC_029", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where IN(1,2)""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -352,15 +304,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check AND with using booleans in invalid syntax
test("Batch_sort_Querying_001-01-01-01_001-TC_034", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where AND true""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -384,15 +330,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check AND using 0 and 1 treated as boolean values
test("Batch_sort_Querying_001-01-01-01_001-TC_037", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where true aNd 0""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -416,29 +356,17 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check '='operator without Passing any value
test("Batch_sort_Querying_001-01-01-01_001-TC_040", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id=""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '='operator without Passing columnname and value.
test("Batch_sort_Querying_001-01-01-01_001-TC_041", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where =""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -453,15 +381,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check '!='operator by keeping space between them
test("Batch_sort_Querying_001-01-01-01_001-TC_043", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id ! = 9001""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -476,29 +398,17 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check '!='operator without providing any value
test("Batch_sort_Querying_001-01-01-01_001-TC_045", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id != """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '!='operator without providing any column name
test("Batch_sort_Querying_001-01-01-01_001-TC_046", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where != false""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -540,43 +450,25 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check 'NOT' operator in nested way
test("Batch_sort_Querying_001-01-01-01_001-TC_051", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id NOT (NOT(true))""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'NOT' operator with parenthesis.
test("Batch_sort_Querying_001-01-01-01_001-TC_052", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id NOT ()""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'NOT' operator without condition.
test("Batch_sort_Querying_001-01-01-01_001-TC_053", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id NOT""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -591,29 +483,17 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check '>' operator without specifying column
test("Batch_sort_Querying_001-01-01-01_001-TC_055", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where > 20""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '>' operator without specifying value
test("Batch_sort_Querying_001-01-01-01_001-TC_056", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id > """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -646,15 +526,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check '<' operator without specifying column
test("Batch_sort_Querying_001-01-01-01_001-TC_060", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where < 5""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -678,29 +552,17 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check '<=' operator without specifying column
test("Batch_sort_Querying_001-01-01-01_001-TC_063", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where <= 2""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check '<=' operator without providing value
test("Batch_sort_Querying_001-01-01-01_001-TC_064", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id <= """).collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -715,29 +577,17 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check '<=' operator adding space between'<' and '='
test("Batch_sort_Querying_001-01-01-01_001-TC_066", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id < = 9002""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'BETWEEN' operator without providing range
test("Batch_sort_Querying_001-01-01-01_001-TC_067", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where age between""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -797,29 +647,17 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check 'IS NULL' without providing column
test("Batch_sort_Querying_001-01-01-01_001-TC_074", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where Is NulL""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check 'IS NOT NULL' without providing column
test("Batch_sort_Querying_001-01-01-01_001-TC_075", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where IS NOT NULL""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -852,29 +690,17 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check Limit clause with where condition and no argument
test("Batch_sort_Querying_001-01-01-01_001-TC_079", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id=10987 limit""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check Limit clause with where condition and decimal argument
test("Batch_sort_Querying_001-01-01-01_001-TC_080", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id=10987 limit 0.0""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -927,15 +753,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check Full join
test("Batch_sort_Querying_001-01-01-01_001-TC_086", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select uniqdataquery1.CUST_ID from uniqdataquery1 FULL JOIN uniqdataquery11 where CUST_ID""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1022,15 +842,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check SORT using 'AND' on multiple column
test("Batch_sort_Querying_001-01-01-01_001-TC_097", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 where cust_id > 10544 sort by cust_name desc and cust_id asc""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1054,15 +868,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check average aggregate function with no arguments
test("Batch_sort_Querying_001-01-01-01_001-TC_100", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_id,avg() from uniqdataquery1 group by cust_id""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1077,15 +885,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check nested average aggregate function
test("Batch_sort_Querying_001-01-01-01_001-TC_102", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_id,avg(count(cust_id)) from uniqdataquery1 group by cust_id""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1172,15 +974,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check Order by without column name
test("Batch_sort_Querying_001-01-01-01_001-TC_112", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select * from uniqdataquery1 order by ASC""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1222,15 +1018,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check Using window without partition
test("Batch_sort_Querying_001-01-01-01_001-TC_117", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_name, sum(bigint_column1) OVER w from uniqdataquery1 WINDOW w""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -1245,15 +1035,9 @@ class BatchSortQueryTestCase extends QueryTest with BeforeAndAfterAll {
//To check Using ROLLUP without group by clause
test("Batch_sort_Querying_001-01-01-01_001-TC_119", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""select cust_name from uniqdataquery1 with ROLLUP""").collect
-
- assert(false)
- } catch {
- case _ => assert(true)
}
- sql(s"""drop table uniqdataquery1""").collect
+ sql(s"""drop table uniqdataquery1""").collect
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BucketingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BucketingTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BucketingTestCase.scala
index 78f8945..501b089 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BucketingTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/BucketingTestCase.scala
@@ -40,28 +40,20 @@ class BucketingTestCase extends QueryTest with BeforeAndAfterAll {
}
test("test exception if bucketcolumns be measure column") {
- try {
+ intercept[Exception] {
sql("DROP TABLE IF EXISTS bucket_table")
sql("CREATE TABLE bucket_table (ID Int, date Timestamp, country String, name String, phonetype String," +
"serialname String, salary Int) STORED BY 'carbondata' TBLPROPERTIES " +
"('BUCKETNUMBER'='4', 'BUCKETCOLUMNS'='ID')")
- assert(false)
- }
- catch {
- case _ => assert(true)
}
}
test("test exception if bucketcolumns be complex data type column") {
- try {
+ intercept[Exception] {
sql("DROP TABLE IF EXISTS bucket_table")
sql("CREATE TABLE bucket_table (Id int, number double, name string, " +
"gamePoint array<double>, mac struct<num:double>) STORED BY 'carbondata' TBLPROPERTIES" +
"('BUCKETNUMBER'='4', 'BUCKETCOLUMNS'='gamePoint')")
- assert(false)
- }
- catch {
- case _ => assert(true)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ColumndictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ColumndictTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ColumndictTestCase.scala
index f702254..c8e8f1b 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ColumndictTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ColumndictTestCase.scala
@@ -55,12 +55,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Load using external columndict for CSV having incomplete/wrong data/no data/null data
test("Columndict-TC004", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/data.csv' into table t3 options('COLUMNDICT'='country:$resourcesPath/Data/columndict/inValidData.csv', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -197,12 +194,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Load using external columndict for table with measure and tableproperties(DICTIONARY_EXCLUDE, DICTIONARY_INCLUDE, BLOCKSIZE)
test("Columndict-TC020", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata' TBLPROPERTIES ('TABLE_BLOCKSIZE'= '256 MB','DICTIONARY_EXCLUDE'='country')""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/data.csv' into table t3 options('COLUMNDICT'='country:'resourcesPath/Data/columndict/country.csv', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -210,12 +204,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Columndict parameter name validation
ignore("Columndict-TC021", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata' TBLPROPERTIES ('TABLE_BLOCKSIZE'= '256 MB','DICTIONARY_EXCLUDE'='country')""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/data.csv' into table t3 options('COLUMNDICT'='countries:$resourcesPath/Data/columndict/country.csv', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -223,12 +214,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Columndict parameter value validation
test("Columndict-TC022", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/data.csv' into table t3 options('COLUMNDICT'='salary:$resourcesPath/Data/columndict/country.csv', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -236,12 +224,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Check for data validation in csv(empty/null/wrong data) for all_dictionary_path
ignore("Columndict-TC023", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/inValidData.csv' into table t3 options('ALL_DICTIONARY_PATH'='$resourcesPath/Data/columndict/inValidData.dictionary', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -249,12 +234,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Check for data validation in csv(empty/null/wrong data) for columndict
test("Columndict-TC024", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/inValidData.csv' into table t3 options('COLUMNDICT'='country:'resourcesPath/Data/columndict/inValidData.csv', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -262,12 +244,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Check for validation of external all_dictionary_path folder with incorrect path
test("Columndict-TC025", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/inValidData.csv' into table t3 options('ALL_DICTIONARY_PATH'=''resourcesPath/Data/*.dictionary', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -275,12 +254,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Check for validation of external all_dictionary_path folder with correct path
test("Columndict-TC026", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/inValidData.csv' into table t3 options('ALL_DICTIONARY_PATH'='$resourcesPath/Data/columndict/*.dictionary', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -288,12 +264,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Check for validation of external columndict folder with correct path
test("Columndict-TC027", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/inValidData.csv' into table t3 options('COLUMNDICT'='country:'resourcesPath/Data/columndict/*.csv', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -301,12 +274,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Check for validation of external all_dictionary_path file( missing /wrong path / wrong name)
test("Columndict-TC028", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/inValidData.csv' into table t3 options('ALL_DICTIONARY_PATH'=''resourcesPath/Data/columndict/wrongName.dictionary', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -314,12 +284,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Check for validation of external columndict file( missing /wrong path / wrong name)
test("Columndict-TC029", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/inValidData.csv' into table t3 options('COLUMNDICT'='country:'resourcesPath/Data/columndict/wrongName.csv', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
@@ -335,12 +302,9 @@ class ColumndictTestCase extends QueryTest with BeforeAndAfterAll {
//Check for different dictionary file extensions for columndict
test("Columndict-TC031", Include) {
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE IF NOT EXISTS t3 (ID Int, country String, name String, phonetype String, serialname String, salary Int,floatField float) STORED BY 'carbondata'""").collect
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/Data/columndict/inValidData.csv' into table t3 options('COLUMNDICT'='country:$resourcesPath/Data/columndict/country.txt', 'SINGLE_PASS'='true')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table if exists t3""").collect
}
[31/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index fdb5310..b40a8e2 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -30,7 +30,7 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -44,7 +44,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import mockit.Mock;
import mockit.MockUp;
@@ -266,8 +266,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
- FixedLengthDimensionDataChunk fixedLengthDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+ FixedLengthDimensionColumnPage fixedLengthDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues);
assertEquals(result, 1);
}
@@ -275,8 +275,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValueToTarget() {
byte[] dataChunks = { 7, 7, 7, 8, 9 };
byte[] compareValues = { 7 };
- FixedLengthDimensionDataChunk fixedLengthDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+ FixedLengthDimensionColumnPage fixedLengthDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues);
assertEquals(result, -1);
}
@@ -284,8 +284,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
- FixedLengthDimensionDataChunk fixedLengthDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+ FixedLengthDimensionColumnPage fixedLengthDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5);
assertEquals(result, 3);
}
@@ -301,8 +301,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValueToTarget() {
byte[] dataChunks = { 5, 6, 7, 7, 7 };
byte[] compareValues = { 7 };
- FixedLengthDimensionDataChunk fixedLengthDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+ FixedLengthDimensionColumnPage fixedLengthDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5);
assertEquals(result, 5);
}
@@ -524,23 +524,23 @@ public class CarbonUtilTest {
}
@Test public void testToGetDictionaryEncodingArray() {
- QueryDimension column1 = new QueryDimension("Column1");
- QueryDimension column2 = new QueryDimension("Column2");
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
List<Encoding> encoding = new ArrayList<>();
encoding.add(Encoding.DICTIONARY);
column1Schema.setEncodingList(encoding);
- column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
column2Schema.setColumnName("Column2");
List<Encoding> encoding2 = new ArrayList<>();
encoding2.add(Encoding.DELTA);
column2Schema.setEncodingList(encoding2);
- column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
- QueryDimension[] queryDimensions = { column1, column2 };
+ ProjectionDimension[] queryDimensions = { column1, column2 };
boolean[] dictionaryEncoding = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
boolean[] expectedDictionaryEncoding = { true, false };
@@ -550,23 +550,23 @@ public class CarbonUtilTest {
}
@Test public void testToGetDirectDictionaryEncodingArray() {
- QueryDimension column1 = new QueryDimension("Column1");
- QueryDimension column2 = new QueryDimension("Column2");
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
List<Encoding> encoding = new ArrayList<>();
encoding.add(Encoding.DIRECT_DICTIONARY);
column1Schema.setEncodingList(encoding);
- column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
column2Schema.setColumnName("Column2");
List<Encoding> encoding2 = new ArrayList<>();
encoding2.add(Encoding.DELTA);
column2Schema.setEncodingList(encoding2);
- column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
- QueryDimension[] queryDimensions = { column1, column2 };
+ ProjectionDimension[] queryDimensions = { column1, column2 };
boolean[] dictionaryEncoding = CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
boolean[] expectedDictionaryEncoding = { true, false };
@@ -576,19 +576,19 @@ public class CarbonUtilTest {
}
@Test public void testToGetComplexDataTypeArray() {
- QueryDimension column1 = new QueryDimension("Column1");
- QueryDimension column2 = new QueryDimension("Column2");
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
column1Schema.setDataType(DataTypes.DATE);
- column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
column2Schema.setColumnName("Column2");
column2Schema.setDataType(DataTypes.createDefaultArrayType());
- column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
- QueryDimension[] queryDimensions = { column1, column2 };
+ ProjectionDimension[] queryDimensions = { column1, column2 };
boolean[] dictionaryEncoding = CarbonUtil.getComplexDataTypeArray(queryDimensions);
boolean[] expectedDictionaryEncoding = { false, true };
@@ -805,8 +805,8 @@ public class CarbonUtilTest {
@Test public void testToGetFirstIndexUsingBinarySearchWithCompareTo1() {
byte[] dataChunks = { 10, 20, 30, 40, 50, 60 };
byte[] compareValue = { 5 };
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 6, 1);
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
assertEquals(-2, result);
@@ -815,8 +815,8 @@ public class CarbonUtilTest {
@Test public void testToGetFirstIndexUsingBinarySearchWithCompareToLessThan0() {
byte[] dataChunks = { 10, 20, 30, 40, 50, 60 };
byte[] compareValue = { 30 };
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 6, 1);
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
assertEquals(2, result);
@@ -825,8 +825,8 @@ public class CarbonUtilTest {
@Test public void testToGetFirstIndexUsingBinarySearchWithCompareTo0() {
byte[] dataChunks = { 10, 10, 10, 40, 50, 60 };
byte[] compareValue = { 10 };
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 6, 1);
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
assertEquals(0, result);
@@ -835,8 +835,8 @@ public class CarbonUtilTest {
@Test public void testToGetFirstIndexUsingBinarySearchWithMatchUpLimitTrue() {
byte[] dataChunks = { 10, 10, 10, 40, 50, 60 };
byte[] compareValue = { 10 };
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 6, 1);
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, true);
assertEquals(2, result);
@@ -846,13 +846,13 @@ public class CarbonUtilTest {
public void testBinaryRangeSearch() {
byte[] dataChunk = new byte[10];
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk;
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
byte[] keyWord = new byte[1];
int[] range;
dataChunk = "abbcccddddeffgggh".getBytes();
byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
@@ -884,7 +884,7 @@ public class CarbonUtilTest {
assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
dataChunk = "ab".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("97");
@@ -898,7 +898,7 @@ public class CarbonUtilTest {
assertEquals(1, range[1]);
dataChunk = "aabb".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("97");
@@ -912,7 +912,7 @@ public class CarbonUtilTest {
assertEquals(3, range[1]);
dataChunk = "a".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("97");
@@ -921,7 +921,7 @@ public class CarbonUtilTest {
assertEquals(0, range[1]);
dataChunk = "aa".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("97");
@@ -930,7 +930,7 @@ public class CarbonUtilTest {
assertEquals(1, range[1]);
dataChunk = "aabbbbbbbbbbcc".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("98");
range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
@@ -943,14 +943,14 @@ public class CarbonUtilTest {
public void IndexUsingBinarySearchLengthTwo() {
byte[] dataChunk = new byte[10];
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk;
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
byte[] keyWord = new byte[2];
dataChunk = "aabbbbbbbbbbcc".getBytes();
byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
@@ -985,14 +985,14 @@ public class CarbonUtilTest {
public void IndexUsingBinarySearchLengthThree() {
byte[] dataChunk = new byte[10];
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk;
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
byte[] keyWord = new byte[3];
dataChunk = "aaabbbbbbbbbccc".getBytes();
byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
@@ -1054,7 +1054,7 @@ public class CarbonUtilTest {
}
private void assertRangeIndex(byte[][] dataArr, byte[] dataChunk,
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk, byte[] keyWord, int[] expectRangeIndex) {
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk, byte[] keyWord, int[] expectRangeIndex) {
int[] range;
range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0,
(dataChunk.length - 1) / keyWord.length, keyWord);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
index de64c0a..e506994 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
@@ -25,10 +25,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.datastore.impl.FileHolderImpl;
+import org.apache.carbondata.core.datastore.impl.FileReaderImpl;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
@@ -229,13 +229,13 @@ public class DataFileFooterConverterTest {
}
@SuppressWarnings("unused") @Mock
- public FileHolder getFileHolder(FileFactory.FileType fileType) {
- return new FileHolderImpl();
+ public FileReader getFileHolder(FileFactory.FileType fileType) {
+ return new FileReaderImpl();
}
};
- new MockUp<FileHolderImpl>() {
+ new MockUp<FileReaderImpl>() {
@SuppressWarnings("unused") @Mock public long readLong(String filePath, long offset) {
return 1;
}
@@ -249,7 +249,6 @@ public class DataFileFooterConverterTest {
SegmentInfo segmentInfo = new SegmentInfo();
int[] arr = { 1, 2, 3 };
segmentInfo.setColumnCardinality(arr);
- segmentInfo.setNumberOfColumns(segmentInfo1.getNum_cols());
dataFileFooter.setNumberOfRows(3);
dataFileFooter.setSegmentInfo(segmentInfo);
TableBlockInfo info = new TableBlockInfo("/file.carbondata", 1, "0", new String[0], 1, ColumnarFormatVersion.V1, null);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
index 4c9a784..4fb5dcc 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
@@ -36,7 +36,6 @@ import org.apache.carbondata.core.scan.expression.logical.RangeExpression;
import org.apache.carbondata.core.scan.expression.logical.TrueExpression;
import org.apache.carbondata.core.scan.filter.executer.RangeValueFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
-import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerBasic;
import org.apache.carbondata.core.scan.filter.optimizer.RangeFilterOptmizer;
import mockit.Deencapsulation;
@@ -102,7 +101,7 @@ public class RangeFilterProcessorTest {
new LessThanEqualToExpression(new ColumnExpression("a", DataTypes.STRING),
new LiteralExpression("20", DataTypes.STRING))), new TrueExpression(null));
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ new RangeFilterOptmizer(inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, output);
Assert.assertTrue(result);
@@ -143,7 +142,7 @@ public class RangeFilterProcessorTest {
new LessThanEqualToExpression(new ColumnExpression("a", DataTypes.STRING),
new LiteralExpression("05", DataTypes.STRING)));
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ new RangeFilterOptmizer(inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, output);
// no change
@@ -218,7 +217,7 @@ public class RangeFilterProcessorTest {
Expression Andb3 = new AndExpression(Andb2, new TrueExpression(null));
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ new RangeFilterOptmizer(inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, new AndExpression(Andb3, new TrueExpression(null)));
// no change
@@ -302,7 +301,7 @@ public class RangeFilterProcessorTest {
Expression Orb3 = new OrExpression(Orb2, lessThanb2);
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ new RangeFilterOptmizer(inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, new OrExpression(Orb3, lessThanb1));
// no change
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java b/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
deleted file mode 100644
index 94c3f68..0000000
--- a/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.scanner.impl;
-
-import static junit.framework.TestCase.assertEquals;
-
-public class FilterScannerTest {
-//
-// private static FilterScanner filterScanner;
-// private static BlockletIndex blockletIndex;
-// private static BlockletMinMaxIndex blockletMinMaxIndex;
-// private static BTreeBuilderInfo bTreeBuilderInfo;
-// private static DataFileFooter dataFileFooter;
-//
-// @BeforeClass public static void setUp() {
-// BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
-// FilterExecuter filterExecutor = new AndFilterExecuterImpl(null, null);
-// blockExecutionInfo.setFilterExecuterTree(filterExecutor);
-// blockExecutionInfo.setFixedLengthKeySize(1);
-// blockExecutionInfo.setNoDictionaryBlockIndexes(new int[] { 1, 2 });
-// blockExecutionInfo.setDictionaryColumnBlockIndex(new int[] { 1 });
-// blockExecutionInfo.setColumnGroupToKeyStructureInfo(new HashMap<Integer, KeyStructureInfo>());
-// blockExecutionInfo.setComplexDimensionInfoMap(new HashMap<Integer, GenericQueryType>());
-// blockExecutionInfo.setComplexColumnParentBlockIndexes(new int[] { 1 });
-// blockExecutionInfo.setQueryDimensions(new QueryDimension[] { new QueryDimension("Col1") });
-// blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[][] { { 0, 0 } });
-// blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[][] { { 0, 0 } });
-// blockExecutionInfo.setTotalNumberOfMeasureBlock(1);
-// blockExecutionInfo.setTotalNumberDimensionBlock(1);
-// QueryStatisticsModel queryStatisticsModel = new QueryStatisticsModel();
-// QueryStatistic queryStatistic = new QueryStatistic();
-// queryStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, 1);
-// Map<String, QueryStatistic> statisticsTypeAndObjMap = new HashMap<>();
-// statisticsTypeAndObjMap.put(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, queryStatistic);
-// statisticsTypeAndObjMap.put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatistic);
-// queryStatisticsModel.setStatisticsTypeAndObjMap(statisticsTypeAndObjMap);
-// QueryStatisticsRecorder queryStatisticsRecorder = new QueryStatisticsRecorderImpl("1");
-// queryStatisticsModel.setRecorder(queryStatisticsRecorder);
-// filterScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel);
-// blockletIndex = new BlockletIndex();
-// blockletMinMaxIndex = new BlockletMinMaxIndex();
-// blockletMinMaxIndex.setMinValues(new byte[][] { { 1, 2 } });
-// blockletMinMaxIndex.setMaxValues(new byte[][] { { 10, 12 } });
-// blockletIndex.setMinMaxIndex(blockletMinMaxIndex);
-// dataFileFooter = new DataFileFooter();
-// dataFileFooter.setBlockletIndex(blockletIndex);
-// bTreeBuilderInfo = new BTreeBuilderInfo(Arrays.asList(dataFileFooter), new int[] { 1 });
-// }
-//
-// @Test public void testToScanBlockletWithEmptyBitSet() throws QueryExecutionException {
-// new MockUp<AndFilterExecuterImpl>() {
-// @SuppressWarnings("unused") @Mock
-// public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-// return new BitSet();
-// }
-// };
-// BlocksChunkHolder blocksChunkHolder = new BlocksChunkHolder(1, 1);
-// DataRefNode dataRefNode = new BlockBTreeLeafNode(bTreeBuilderInfo, 0, 1);
-// blocksChunkHolder.setDataBlock(dataRefNode);
-// AbstractScannedResult abstractScannedResult = filterScanner.scanBlocklet(blocksChunkHolder);
-// assertEquals(0, abstractScannedResult.numberOfOutputRows());
-// }
-//
-// @Test public void testToScanBlockletWithNonEmptyBitSet() throws QueryExecutionException {
-// new MockUp<AndFilterExecuterImpl>() {
-// @SuppressWarnings("unused") @Mock
-// public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// bitSet.set(2);
-// bitSet.set(1);
-// return bitSet;
-// }
-//
-// @SuppressWarnings("unused") @Mock
-// public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
-// throws FilterUnsupportedException {
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// bitSet.set(2);
-// bitSet.set(1);
-// return bitSet;
-// }
-// };
-// DataRefNode dataRefNode = new MockUp<DataRefNode>() {
-// @Mock @SuppressWarnings("unused") DimensionColumnDataChunk[] getDimensionChunks(
-// FileHolder fileReader, int[][] blockIndexes) {
-// DimensionColumnDataChunk[] dimensionChunkAttributes =
-// { new ColumnGroupDimensionDataChunk(null, null) };
-// return dimensionChunkAttributes;
-// }
-//
-// @Mock @SuppressWarnings("unused") ColumnPage[] getMeasureChunks(
-// FileHolder fileReader, int[][] blockIndexes) {
-//
-// ColumnPage[] ColumnPages = { new ColumnPage() };
-// return ColumnPages;
-// }
-// }.getMockInstance();
-//
-// BlocksChunkHolder blocksChunkHolder = new BlocksChunkHolder(1, 1);
-// blocksChunkHolder.setDataBlock(dataRefNode);
-// DimensionChunkAttributes dimensionChunkAttributes = new DimensionChunkAttributes();
-// DimensionColumnDataChunk dimensionColumnDataChunk =
-// new FixedLengthDimensionDataChunk(new byte[] { 0, 1 }, dimensionChunkAttributes);
-// blocksChunkHolder.setDimensionRawDataChunk(new DimensionColumnDataChunk[]
-//
-// { dimensionColumnDataChunk });
-// ColumnPage ColumnPage = new ColumnPage();
-// blocksChunkHolder.setMeasureDataChunk(new ColumnPage[]
-//
-// { ColumnPage });
-// FileHolder fileHolder = new DFSFileHolderImpl();
-// blocksChunkHolder.setFileReader(fileHolder);
-// AbstractScannedResult abstractScannedResult = filterScanner.scanBlocklet(blocksChunkHolder);
-//
-// assertEquals(2, abstractScannedResult.numberOfOutputRows());
-// }
-//
-// @Test(expected = QueryExecutionException.class) public void testToScanBlockletWithException()
-// throws QueryExecutionException {
-// new MockUp<AndFilterExecuterImpl>() {
-// @SuppressWarnings("unused") @Mock
-// public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// bitSet.set(2);
-// bitSet.set(1);
-// return bitSet;
-// }
-//
-// @SuppressWarnings("unused") @Mock
-// public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
-// throws FilterUnsupportedException {
-// throw new FilterUnsupportedException("Filter unsupported");
-// }
-// };
-// BlocksChunkHolder blocksChunkHolder = new BlocksChunkHolder(1, 1);
-// BTreeBuilderInfo bTreeBuilderInfo =
-// new BTreeBuilderInfo(Arrays.asList(dataFileFooter), new int[] { 1 });
-// DataRefNode dataRefNode = new BlockBTreeLeafNode(bTreeBuilderInfo, 0, 1);
-// blocksChunkHolder.setDataBlock(dataRefNode);
-// filterScanner.scanBlocklet(blocksChunkHolder);
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/dev/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/dev/findbugs-exclude.xml b/dev/findbugs-exclude.xml
index 1520cd4..b19db85 100644
--- a/dev/findbugs-exclude.xml
+++ b/dev/findbugs-exclude.xml
@@ -31,7 +31,7 @@
<Bug pattern="OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE"/>
</Match>
<Match>
- <Class name="org.apache.carbondata.core.datastore.impl.FileHolderImpl"/>
+ <Class name="org.apache.carbondata.core.datastore.impl.FileReaderImpl"/>
<Method name="getDataInputStream"/>
<Bug pattern="OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE"/>
</Match>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index 76afcbf..7a15327 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -62,7 +62,7 @@ object CarbonSessionExample {
spark.sql(
s"""
- | SELECT *
+ | SELECT charField, stringField, intField
| FROM carbon_table
| WHERE stringfield = 'spark' AND decimalField > 40
""".stripMargin).show()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 6f1e123..b1faa6a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -57,7 +57,6 @@ import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
import org.apache.carbondata.core.scan.filter.SingleTableProvider;
import org.apache.carbondata.core.scan.filter.TableProvider;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
@@ -110,11 +109,11 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// comma separated list of input segment numbers
public static final String INPUT_SEGMENT_NUMBERS =
"mapreduce.input.carboninputformat.segmentnumbers";
- public static final String VALIDATE_INPUT_SEGMENT_IDs =
+ private static final String VALIDATE_INPUT_SEGMENT_IDs =
"mapreduce.input.carboninputformat.validsegments";
// comma separated list of input files
public static final String INPUT_FILES = "mapreduce.input.carboninputformat.files";
- public static final String ALTER_PARTITION_ID = "mapreduce.input.carboninputformat.partitionid";
+ private static final String ALTER_PARTITION_ID = "mapreduce.input.carboninputformat.partitionid";
private static final Log LOG = LogFactory.getLog(CarbonTableInputFormat.class);
private static final String FILTER_PREDICATE =
"mapreduce.input.carboninputformat.filter.predicate";
@@ -125,7 +124,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
private static final String DATA_MAP_DSTR = "mapreduce.input.carboninputformat.datamapdstr";
public static final String DATABASE_NAME = "mapreduce.input.carboninputformat.databaseName";
public static final String TABLE_NAME = "mapreduce.input.carboninputformat.tableName";
- public static final String PARTITIONS_TO_PRUNE =
+ private static final String PARTITIONS_TO_PRUNE =
"mapreduce.input.carboninputformat.partitions.to.prune";
public static final String UPADTE_T =
"mapreduce.input.carboninputformat.partitions.to.prune";
@@ -307,7 +306,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
/**
* get list of partitions to prune
*/
- public static List<String> getPartitionsToPrune(Configuration configuration) throws IOException {
+ private static List<String> getPartitionsToPrune(Configuration configuration) throws IOException {
String partitionString = configuration.get(PARTITIONS_TO_PRUNE);
if (partitionString != null) {
return (List<String>) ObjectSerializationUtil.convertStringToObject(partitionString);
@@ -335,7 +334,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
* @return List<InputSplit> list of CarbonInputSplit
* @throws IOException
*/
- @Override public List<InputSplit> getSplits(JobContext job) throws IOException {
+ @Override
+ public List<InputSplit> getSplits(JobContext job) throws IOException {
AbsoluteTableIdentifier identifier = getAbsoluteTableIdentifier(job.getConfiguration());
SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier);
CarbonTable carbonTable = getOrCreateCarbonTable(job.getConfiguration());
@@ -789,28 +789,29 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
return split;
}
- @Override public RecordReader<Void, T> createRecordReader(InputSplit inputSplit,
+ @Override
+ public RecordReader<Void, T> createRecordReader(InputSplit inputSplit,
TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
Configuration configuration = taskAttemptContext.getConfiguration();
- QueryModel queryModel = getQueryModel(inputSplit, taskAttemptContext);
+ QueryModel queryModel = createQueryModel(inputSplit, taskAttemptContext);
CarbonReadSupport<T> readSupport = getReadSupportClass(configuration);
return new CarbonRecordReader<T>(queryModel, readSupport);
}
- public QueryModel getQueryModel(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
+ public QueryModel createQueryModel(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
throws IOException {
Configuration configuration = taskAttemptContext.getConfiguration();
CarbonTable carbonTable = getOrCreateCarbonTable(configuration);
TableProvider tableProvider = new SingleTableProvider(carbonTable);
- // getting the table absoluteTableIdentifier from the carbonTable
- // to avoid unnecessary deserialization
- AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
// query plan includes projection column
- String projection = getColumnProjection(configuration);
- CarbonQueryPlan queryPlan = CarbonInputFormatUtil.createQueryPlan(carbonTable, projection);
- QueryModel queryModel = QueryModel.createModel(identifier, queryPlan, carbonTable,
- getDataTypeConverter(configuration));
+ String projectionString = getColumnProjection(configuration);
+ String[] projectionColumnNames = null;
+ if (projectionString != null) {
+ projectionColumnNames = projectionString.split(",");
+ }
+ QueryModel queryModel = carbonTable.createQueryWithProjection(
+ projectionColumnNames, getDataTypeConverter(configuration));
// set the filter to the query model in order to filter blocklet before scan
Expression filter = getFilterPredicates(configuration);
@@ -865,7 +866,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
return readSupport;
}
- @Override protected boolean isSplitable(JobContext context, Path filename) {
+ @Override
+ protected boolean isSplitable(JobContext context, Path filename) {
try {
// Don't split the file if it is local file system
FileSystem fileSystem = filename.getFileSystem(context.getConfiguration());
@@ -879,19 +881,9 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
}
/**
- * required to be moved to core
- *
- * @return updateExtension
- */
- private String getUpdateExtension() {
- // TODO: required to modify when supporting update, mostly will be update timestamp
- return "update";
- }
-
- /**
* return valid segment to access
*/
- public String[] getSegmentsToAccess(JobContext job) {
+ private String[] getSegmentsToAccess(JobContext job) {
String segmentString = job.getConfiguration().get(INPUT_SEGMENT_NUMBERS, "");
if (segmentString.trim().isEmpty()) {
return new String[0];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index a590a5b..0fe0cbf 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -176,9 +176,7 @@ class InMemoryBTreeIndex implements Index {
filterredBlocks = filterExpressionProcessor.getFilterredBlocks(
abstractIndex.getDataRefNode(),
resolver,
- abstractIndex,
- identifier
- );
+ abstractIndex);
}
resultFilterredBlocks.addAll(filterredBlocks);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
index 19626f0..e7c6dda 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
@@ -152,7 +152,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
hadoopConf = context.getConfiguration();
if (model == null) {
CarbonTableInputFormat format = new CarbonTableInputFormat<Object>();
- model = format.getQueryModel(split, context);
+ model = format.createQueryModel(split, context);
}
carbonTable = model.getTable();
List<CarbonDimension> dimensions =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index 89a4a9a..2f28861 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -67,7 +67,7 @@ public class BlockLevelTraverser {
blockName = CarbonTablePath.getCarbonDataFileName(blockName);
blockName = blockName + CarbonTablePath.getCarbonDataExtension();
- long rowCount = currentBlock.nodeSize();
+ long rowCount = currentBlock.numRows();
String key = CarbonUpdateUtil.getSegmentBlockNameKey(segId, blockName);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 056c27b..9f8c5ec 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -26,18 +26,12 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
import org.apache.carbondata.core.scan.filter.TableProvider;
import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
-import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerBasic;
import org.apache.carbondata.core.scan.filter.optimizer.RangeFilterOptmizer;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
@@ -52,45 +46,14 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
*/
public class CarbonInputFormatUtil {
- public static CarbonQueryPlan createQueryPlan(CarbonTable carbonTable, String columnString) {
- String[] columns = null;
- if (columnString != null) {
- columns = columnString.split(",");
- }
- String factTableName = carbonTable.getTableName();
- CarbonQueryPlan plan = new CarbonQueryPlan(carbonTable.getDatabaseName(), factTableName);
- // fill dimensions
- // If columns are null, set all dimensions and measures
- int i = 0;
- if (columns != null) {
- for (String column : columns) {
- CarbonDimension dimensionByName = carbonTable.getDimensionByName(factTableName, column);
- if (dimensionByName != null) {
- addQueryDimension(plan, i, dimensionByName);
- i++;
- } else {
- CarbonMeasure measure = carbonTable.getMeasureByName(factTableName, column);
- if (measure == null) {
- throw new RuntimeException(column + " column not found in the table " + factTableName);
- }
- addQueryMeasure(plan, i, measure);
- i++;
- }
- }
- }
-
- plan.setQueryId(System.nanoTime() + "");
- return plan;
- }
-
public static <V> CarbonTableInputFormat<V> createCarbonInputFormat(
AbsoluteTableIdentifier identifier,
Job job) throws IOException {
CarbonTableInputFormat<V> carbonInputFormat = new CarbonTableInputFormat<>();
- carbonInputFormat.setDatabaseName(job.getConfiguration(),
- identifier.getCarbonTableIdentifier().getDatabaseName());
- carbonInputFormat
- .setTableName(job.getConfiguration(), identifier.getCarbonTableIdentifier().getTableName());
+ CarbonTableInputFormat.setDatabaseName(
+ job.getConfiguration(), identifier.getCarbonTableIdentifier().getDatabaseName());
+ CarbonTableInputFormat.setTableName(
+ job.getConfiguration(), identifier.getCarbonTableIdentifier().getTableName());
FileInputFormat.addInputPath(job, new Path(identifier.getTablePath()));
return carbonInputFormat;
}
@@ -98,30 +61,16 @@ public class CarbonInputFormatUtil {
public static <V> CarbonTableInputFormat<V> createCarbonTableInputFormat(
AbsoluteTableIdentifier identifier, List<String> partitionId, Job job) throws IOException {
CarbonTableInputFormat<V> carbonTableInputFormat = new CarbonTableInputFormat<>();
- carbonTableInputFormat.setPartitionIdList(job.getConfiguration(), partitionId);
- carbonTableInputFormat.setDatabaseName(job.getConfiguration(),
- identifier.getCarbonTableIdentifier().getDatabaseName());
- carbonTableInputFormat
- .setTableName(job.getConfiguration(), identifier.getCarbonTableIdentifier().getTableName());
+ CarbonTableInputFormat.setPartitionIdList(
+ job.getConfiguration(), partitionId);
+ CarbonTableInputFormat.setDatabaseName(
+ job.getConfiguration(), identifier.getCarbonTableIdentifier().getDatabaseName());
+ CarbonTableInputFormat.setTableName(
+ job.getConfiguration(), identifier.getCarbonTableIdentifier().getTableName());
FileInputFormat.addInputPath(job, new Path(identifier.getTablePath()));
return carbonTableInputFormat;
}
- private static void addQueryMeasure(CarbonQueryPlan plan, int order, CarbonMeasure measure) {
- QueryMeasure queryMeasure = new QueryMeasure(measure.getColName());
- queryMeasure.setQueryOrder(order);
- queryMeasure.setMeasure(measure);
- plan.addMeasure(queryMeasure);
- }
-
- private static void addQueryDimension(CarbonQueryPlan plan, int order,
- CarbonDimension dimension) {
- QueryDimension queryDimension = new QueryDimension(dimension.getColName());
- queryDimension.setQueryOrder(order);
- queryDimension.setDimension(dimension);
- plan.addDimension(queryDimension);
- }
-
public static void processFilterExpression(Expression filterExpression, CarbonTable carbonTable,
boolean[] isFilterDimensions, boolean[] isFilterMeasures) {
QueryModel.processFilterExpression(carbonTable, filterExpression, isFilterDimensions,
@@ -130,7 +79,7 @@ public class CarbonInputFormatUtil {
if (null != filterExpression) {
// Optimize Filter Expression and fit RANGE filters is conditions apply.
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), filterExpression);
+ new RangeFilterOptmizer(filterExpression);
rangeFilterOptimizer.optimizeFilter();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
index f109e1c..1b57f93 100644
--- a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
+++ b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.SingleTableProvider;
import org.apache.carbondata.core.scan.filter.TableProvider;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.util.DataTypeConverterImpl;
import org.apache.carbondata.hadoop.CarbonInputSplit;
@@ -140,11 +139,11 @@ public class MapredCarbonInputFormat extends CarbonTableInputFormat<ArrayWritabl
AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
- String projection = getProjection(configuration, carbonTable,
+ String projectionString = getProjection(configuration, carbonTable,
identifier.getCarbonTableIdentifier().getTableName());
- CarbonQueryPlan queryPlan = CarbonInputFormatUtil.createQueryPlan(carbonTable, projection);
- QueryModel queryModel =
- QueryModel.createModel(identifier, queryPlan, carbonTable, new DataTypeConverterImpl());
+ String[] projectionColumns = projectionString.split(",");
+ QueryModel queryModel = carbonTable.createQueryWithProjection(
+ projectionColumns, new DataTypeConverterImpl());
// set the filter to the query model in order to filter blocklet before scan
Expression filter = getFilterPredicates(configuration);
CarbonInputFormatUtil.processFilterExpression(filter, carbonTable, null, null);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
deleted file mode 100644
index 9a8f8c5..0000000
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.presto;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.datatype.StructField;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
-import org.apache.carbondata.core.scan.executor.QueryExecutor;
-import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
-import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.hadoop.AbstractRecordReader;
-import org.apache.carbondata.hadoop.CarbonInputSplit;
-import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
-
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-/**
- * A specialized RecordReader that reads into InternalRows or ColumnarBatches directly using the
- * carbondata column APIs and fills the data directly into columns.
- */
-class CarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
-
- private int batchIdx = 0;
-
- private int numBatched = 0;
-
- private CarbonVectorBatch columnarBatch;
-
- private CarbonColumnarBatch carbonColumnarBatch;
-
- /**
- * If true, this class returns batches instead of rows.
- */
- private boolean returnColumnarBatch;
-
- private QueryModel queryModel;
-
- private AbstractDetailQueryResultIterator iterator;
-
- private QueryExecutor queryExecutor;
-
- public CarbonVectorizedRecordReader(QueryExecutor queryExecutor, QueryModel queryModel, AbstractDetailQueryResultIterator iterator) {
- this.queryModel = queryModel;
- this.iterator = iterator;
- this.queryExecutor = queryExecutor;
- enableReturningBatches();
- }
-
- /**
- * Implementation of RecordReader API.
- */
- @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
- throws IOException, InterruptedException, UnsupportedOperationException {
- // The input split can contain single HDFS block or multiple blocks, so firstly get all the
- // blocks and then set them in the query model.
- List<CarbonInputSplit> splitList;
- if (inputSplit instanceof CarbonInputSplit) {
- splitList = new ArrayList<>(1);
- splitList.add((CarbonInputSplit) inputSplit);
- } else if (inputSplit instanceof CarbonMultiBlockSplit) {
- // contains multiple blocks, this is an optimization for concurrent query.
- CarbonMultiBlockSplit multiBlockSplit = (CarbonMultiBlockSplit) inputSplit;
- splitList = multiBlockSplit.getAllSplits();
- } else {
- throw new RuntimeException("unsupported input split type: " + inputSplit);
- }
- List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList);
- queryModel.setTableBlockInfos(tableBlockInfoList);
- queryModel.setVectorReader(true);
- try {
- queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
- iterator = (AbstractDetailQueryResultIterator) queryExecutor.execute(queryModel);
- } catch (QueryExecutionException e) {
- throw new InterruptedException(e.getMessage());
- }
- }
-
- @Override public void close() throws IOException {
- logStatistics(rowCount, queryModel.getStatisticsRecorder());
- if (columnarBatch != null) {
- columnarBatch = null;
- }
- // clear dictionary cache
- Map<String, Dictionary> columnToDictionaryMapping = queryModel.getColumnToDictionaryMapping();
- if (null != columnToDictionaryMapping) {
- for (Map.Entry<String, Dictionary> entry : columnToDictionaryMapping.entrySet()) {
- CarbonUtil.clearDictionaryCache(entry.getValue());
- }
- }
- try {
- queryExecutor.finish();
- } catch (QueryExecutionException e) {
- throw new IOException(e);
- }
- }
-
- @Override public boolean nextKeyValue() throws IOException, InterruptedException {
- resultBatch();
-
- if (returnColumnarBatch) return nextBatch();
-
- if (batchIdx >= numBatched) {
- if (!nextBatch()) return false;
- }
- ++batchIdx;
- return true;
- }
-
- @Override public Object getCurrentValue() throws IOException, InterruptedException {
- if (returnColumnarBatch) {
- rowCount += columnarBatch.numValidRows();
- return columnarBatch;
- } else {
- return null;
- }
- }
-
- @Override public Void getCurrentKey() throws IOException, InterruptedException {
- return null;
- }
-
- @Override public float getProgress() throws IOException, InterruptedException {
- // TODO : Implement it based on total number of rows it is going to retrive.
- return 0;
- }
-
- /**
- * Returns the ColumnarBatch object that will be used for all rows returned by this reader.
- * This object is reused. Calling this enables the vectorized reader. This should be called
- * before any calls to nextKeyValue/nextBatch.
- */
-
- private void initBatch() {
- List<QueryDimension> queryDimension = queryModel.getQueryDimension();
- List<QueryMeasure> queryMeasures = queryModel.getQueryMeasures();
- StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
- for (int i = 0; i < queryDimension.size(); i++) {
- QueryDimension dim = queryDimension.get(i);
- if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(dim.getDimension().getDataType());
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
- generator.getReturnType());
- } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
- dim.getDimension().getDataType());
- } else if (dim.getDimension().isComplex()) {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
- dim.getDimension().getDataType());
- } else {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
- DataTypes.INT);
- }
- }
-
- for (QueryMeasure msr : queryMeasures) {
- DataType dataType = msr.getMeasure().getDataType();
- if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT
- || dataType == DataTypes.LONG) {
- fields[msr.getQueryOrder()] =
- new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
- } else if (DataTypes.isDecimal(dataType)) {
- fields[msr.getQueryOrder()] =
- new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
- } else {
- fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE);
- }
- }
-
- columnarBatch = CarbonVectorBatch.allocate(fields);
- CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
- boolean[] filteredRows = new boolean[columnarBatch.capacity()];
- for (int i = 0; i < fields.length; i++) {
- vectors[i] = new CarbonColumnVectorWrapper(columnarBatch.column(i), filteredRows);
- }
- carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
- }
-
-
- private CarbonVectorBatch resultBatch() {
- if (columnarBatch == null) initBatch();
- return columnarBatch;
- }
-
- /*
- * Can be called before any rows are returned to enable returning columnar batches directly.
- */
- private void enableReturningBatches() {
- returnColumnarBatch = true;
- }
-
- /**
- * Advances to the next batch of rows. Returns false if there are no more.
- */
- private boolean nextBatch() {
- columnarBatch.reset();
- carbonColumnarBatch.reset();
- if (iterator.hasNext()) {
- iterator.processNextBatch(carbonColumnarBatch);
- int actualSize = carbonColumnarBatch.getActualSize();
- columnarBatch.setNumRows(actualSize);
- numBatched = actualSize;
- batchIdx = 0;
- return true;
- }
- return false;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
index 1679f29..5f1f90a 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
@@ -54,7 +54,7 @@ class CarbondataPageSource implements ConnectorPageSource {
private final List<Type> types;
private final PageBuilder pageBuilder;
private boolean closed;
- private CarbonVectorizedRecordReader vectorReader;
+ private PrestoCarbonVectorizedRecordReader vectorReader;
private CarbonDictionaryDecodeReadSupport<Object[]> readSupport;
private long sizeOfData = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java
index c614fa9..5772fbf 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java
@@ -54,7 +54,7 @@ public class CarbondataRecordCursor implements RecordCursor {
private CarbondataSplit split;
private CarbonDictionaryDecodeReadSupport readSupport;
private Tuple3<DataType, Dictionary, Int>[] dictionary;
- CarbonVectorizedRecordReader vectorizedRecordReader;
+ PrestoCarbonVectorizedRecordReader vectorizedRecordReader;
private long totalBytes;
private long nanoStart;
@@ -63,7 +63,7 @@ public class CarbondataRecordCursor implements RecordCursor {
public CarbondataRecordCursor(CarbonDictionaryDecodeReadSupport readSupport,
- CarbonVectorizedRecordReader vectorizedRecordReader,
+ PrestoCarbonVectorizedRecordReader vectorizedRecordReader,
List<CarbondataColumnHandle> columnHandles,
CarbondataSplit split) {
this.vectorizedRecordReader = vectorizedRecordReader;
@@ -194,7 +194,7 @@ public class CarbondataRecordCursor implements RecordCursor {
//todo delete cache from readSupport
}
- public CarbonVectorizedRecordReader getVectorizedRecordReader() {
+ public PrestoCarbonVectorizedRecordReader getVectorizedRecordReader() {
return vectorizedRecordReader;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
index 0f8fe87..286ff0e 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
@@ -78,8 +78,8 @@ public class CarbondataRecordSet implements RecordSet {
readSupport
.initialize(queryModel.getProjectionColumns(), queryModel.getTable());
CarbonIterator iterator = queryExecutor.execute(queryModel);
- CarbonVectorizedRecordReader vectorReader =
- new CarbonVectorizedRecordReader(queryExecutor, queryModel,
+ PrestoCarbonVectorizedRecordReader vectorReader =
+ new PrestoCarbonVectorizedRecordReader(queryExecutor, queryModel,
(AbstractDetailQueryResultIterator) iterator);
return new CarbondataRecordCursor(readSupport, vectorReader, columns, split);
} catch (QueryExecutionException e) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
index f039daf..5a2f831 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
@@ -105,7 +105,7 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider {
new TaskAttemptContextImpl(jobConf, new TaskAttemptID("", 1, TaskType.MAP, 0, 0));
CarbonInputSplit carbonInputSplit =
CarbonLocalInputSplit.convertSplit(carbondataSplit.getLocalInputSplit());
- queryModel = carbonTableInputFormat.getQueryModel(carbonInputSplit, hadoopAttemptContext);
+ queryModel = carbonTableInputFormat.createQueryModel(carbonInputSplit, hadoopAttemptContext);
queryModel.setVectorReader(true);
} catch (IOException e) {
throw new RuntimeException("Unable to get the Query Model ", e);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
new file mode 100644
index 0000000..a1907db
--- /dev/null
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.presto;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.StructField;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.scan.executor.QueryExecutor;
+import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.hadoop.AbstractRecordReader;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * A specialized RecordReader that reads into InternalRows or ColumnarBatches directly using the
+ * carbondata column APIs and fills the data directly into columns.
+ */
+class PrestoCarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
+
+ private int batchIdx = 0;
+
+ private int numBatched = 0;
+
+ private CarbonVectorBatch columnarBatch;
+
+ private CarbonColumnarBatch carbonColumnarBatch;
+
+ /**
+ * If true, this class returns batches instead of rows.
+ */
+ private boolean returnColumnarBatch;
+
+ private QueryModel queryModel;
+
+ private AbstractDetailQueryResultIterator iterator;
+
+ private QueryExecutor queryExecutor;
+
+ public PrestoCarbonVectorizedRecordReader(QueryExecutor queryExecutor, QueryModel queryModel, AbstractDetailQueryResultIterator iterator) {
+ this.queryModel = queryModel;
+ this.iterator = iterator;
+ this.queryExecutor = queryExecutor;
+ enableReturningBatches();
+ }
+
+ /**
+ * Implementation of RecordReader API.
+ */
+ @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
+ throws IOException, InterruptedException, UnsupportedOperationException {
+ // The input split can contain single HDFS block or multiple blocks, so firstly get all the
+ // blocks and then set them in the query model.
+ List<CarbonInputSplit> splitList;
+ if (inputSplit instanceof CarbonInputSplit) {
+ splitList = new ArrayList<>(1);
+ splitList.add((CarbonInputSplit) inputSplit);
+ } else if (inputSplit instanceof CarbonMultiBlockSplit) {
+ // contains multiple blocks, this is an optimization for concurrent query.
+ CarbonMultiBlockSplit multiBlockSplit = (CarbonMultiBlockSplit) inputSplit;
+ splitList = multiBlockSplit.getAllSplits();
+ } else {
+ throw new RuntimeException("unsupported input split type: " + inputSplit);
+ }
+ List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList);
+ queryModel.setTableBlockInfos(tableBlockInfoList);
+ queryModel.setVectorReader(true);
+ try {
+ queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
+ iterator = (AbstractDetailQueryResultIterator) queryExecutor.execute(queryModel);
+ } catch (QueryExecutionException e) {
+ throw new InterruptedException(e.getMessage());
+ }
+ }
+
+ @Override public void close() throws IOException {
+ logStatistics(rowCount, queryModel.getStatisticsRecorder());
+ if (columnarBatch != null) {
+ columnarBatch = null;
+ }
+ // clear dictionary cache
+ Map<String, Dictionary> columnToDictionaryMapping = queryModel.getColumnToDictionaryMapping();
+ if (null != columnToDictionaryMapping) {
+ for (Map.Entry<String, Dictionary> entry : columnToDictionaryMapping.entrySet()) {
+ CarbonUtil.clearDictionaryCache(entry.getValue());
+ }
+ }
+ try {
+ queryExecutor.finish();
+ } catch (QueryExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override public boolean nextKeyValue() throws IOException, InterruptedException {
+ resultBatch();
+
+ if (returnColumnarBatch) return nextBatch();
+
+ if (batchIdx >= numBatched) {
+ if (!nextBatch()) return false;
+ }
+ ++batchIdx;
+ return true;
+ }
+
+ @Override public Object getCurrentValue() throws IOException, InterruptedException {
+ if (returnColumnarBatch) {
+ rowCount += columnarBatch.numValidRows();
+ return columnarBatch;
+ } else {
+ return null;
+ }
+ }
+
+ @Override public Void getCurrentKey() throws IOException, InterruptedException {
+ return null;
+ }
+
+ @Override public float getProgress() throws IOException, InterruptedException {
+ // TODO : Implement it based on total number of rows it is going to retrive.
+ return 0;
+ }
+
+ /**
+ * Returns the ColumnarBatch object that will be used for all rows returned by this reader.
+ * This object is reused. Calling this enables the vectorized reader. This should be called
+ * before any calls to nextKeyValue/nextBatch.
+ */
+
+ private void initBatch() {
+ List<ProjectionDimension> queryDimension = queryModel.getProjectionDimensions();
+ List<ProjectionMeasure> queryMeasures = queryModel.getProjectionMeasures();
+ StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
+ for (int i = 0; i < queryDimension.size(); i++) {
+ ProjectionDimension dim = queryDimension.get(i);
+ if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(dim.getDimension().getDataType());
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
+ generator.getReturnType());
+ } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
+ dim.getDimension().getDataType());
+ } else if (dim.getDimension().isComplex()) {
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
+ dim.getDimension().getDataType());
+ } else {
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
+ DataTypes.INT);
+ }
+ }
+
+ for (ProjectionMeasure msr : queryMeasures) {
+ DataType dataType = msr.getMeasure().getDataType();
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT
+ || dataType == DataTypes.LONG) {
+ fields[msr.getOrdinal()] =
+ new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
+ } else if (DataTypes.isDecimal(dataType)) {
+ fields[msr.getOrdinal()] =
+ new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
+ } else {
+ fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE);
+ }
+ }
+
+ columnarBatch = CarbonVectorBatch.allocate(fields);
+ CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
+ boolean[] filteredRows = new boolean[columnarBatch.capacity()];
+ for (int i = 0; i < fields.length; i++) {
+ vectors[i] = new CarbonColumnVectorWrapper(columnarBatch.column(i), filteredRows);
+ }
+ carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
+ }
+
+
+ private CarbonVectorBatch resultBatch() {
+ if (columnarBatch == null) initBatch();
+ return columnarBatch;
+ }
+
+ /*
+ * Can be called before any rows are returned to enable returning columnar batches directly.
+ */
+ private void enableReturningBatches() {
+ returnColumnarBatch = true;
+ }
+
+ /**
+ * Advances to the next batch of rows. Returns false if there are no more.
+ */
+ private boolean nextBatch() {
+ columnarBatch.reset();
+ carbonColumnarBatch.reset();
+ if (iterator.hasNext()) {
+ iterator.processNextBatch(carbonColumnarBatch);
+ int actualSize = carbonColumnarBatch.getActualSize();
+ columnarBatch.setNumRows(actualSize);
+ numBatched = actualSize;
+ batchIdx = 0;
+ return true;
+ }
+ return false;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
index b1fc0a7..95345de 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
@@ -274,17 +274,17 @@ test("Creation of partition table should fail if the colname in table schema and
test("drop partition on preAggregate table should fail"){
sql("drop table if exists partitionTable")
- sql("drop datamap if exists preaggTable on table partitionTable")
sql("create table partitionTable (id int,city string,age int) partitioned by(name string) stored by 'carbondata'".stripMargin)
sql(
- s"""create datamap preaggTable on table partitionTable using 'preaggregate' as select id,sum(age) from partitionTable group by id"""
- .stripMargin)
+ s"""create datamap preaggTable on table partitionTable using 'preaggregate' as select id,sum(age) from partitionTable group by id"""
+ .stripMargin)
sql("insert into partitionTable select 1,'Bangalore',30,'John'")
sql("insert into partitionTable select 2,'Chennai',20,'Huawei'")
checkAnswer(sql("show partitions partitionTable"), Seq(Row("name=John"),Row("name=Huawei")))
intercept[Exception]{
sql("alter table partitionTable drop PARTITION(name='John')")
}
+ sql("drop datamap if exists preaggTable on table partitionTable")
}
@@ -318,7 +318,6 @@ test("Creation of partition table should fail if the colname in table schema and
sql("drop table if exists badrecordsPartitionintnull")
sql("drop table if exists badrecordsPartitionintnullalt")
sql("drop table if exists partitionTable")
- sql("drop datamap if exists preaggTable on table partitionTable")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index a9f8667..0d0f024 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -183,9 +183,9 @@ class CarbonMergerRDD[K, V](
.checkIfAnyRestructuredBlockExists(segmentMapping,
dataFileMetadataSegMapping,
carbonTable.getTableLastUpdatedTime)
- DataTypeUtil.setDataTypeConverter(new SparkDataTypeConverterImpl)
exec = new CarbonCompactionExecutor(segmentMapping, segmentProperties,
- carbonTable, dataFileMetadataSegMapping, restructuredBlockExists)
+ carbonTable, dataFileMetadataSegMapping, restructuredBlockExists,
+ new SparkDataTypeConverterImpl)
// fire a query and get the results.
var result2: java.util.List[RawResultIterator] = null
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 49c0225..084a748 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -331,7 +331,7 @@ class CarbonScanRDD(
TaskMetricsMap.getInstance().registerThreadCallback()
inputMetricsStats.initBytesReadCallback(context, inputSplit)
val iterator = if (inputSplit.getAllSplits.size() > 0) {
- val model = format.getQueryModel(inputSplit, attemptContext)
+ val model = format.createQueryModel(inputSplit, attemptContext)
// get RecordReader by FileFormat
val reader: RecordReader[Void, Object] = inputSplit.getFileFormat match {
case FileFormat.ROW_V1 =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
index 7d42130..432d50a 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
@@ -40,7 +40,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
private DataType blockDataType;
- public ColumnarVectorWrapper(ColumnVector columnVector, boolean[] filteredRows) {
+ ColumnarVectorWrapper(ColumnVector columnVector, boolean[] filteredRows) {
this.columnVector = columnVector;
this.filteredRows = filteredRows;
this.dataType = CarbonScalaUtil.convertSparkToCarbonDataType(columnVector.dataType());
[06/50] [abbrv] carbondata git commit: [CARBONDATA-2100] Add query
test case to check result of streaming handoff operation
Posted by ra...@apache.org.
[CARBONDATA-2100] Add query test case to check result of streaming handoff operation
Add query test case to check result of streaming handoff operation
This closes #1887
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c630b7e0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c630b7e0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c630b7e0
Branch: refs/heads/fgdatamap
Commit: c630b7e0c9b3b0eb53550b5500e7d58904a9376b
Parents: 9b47961
Author: QiangCai <qi...@qq.com>
Authored: Tue Jan 30 18:43:51 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 09:40:32 2018 +0800
----------------------------------------------------------------------
.../org/apache/spark/carbondata/TestStreamingTableOperation.scala | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c630b7e0/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 3de1391..18e52f6 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -527,8 +527,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
Seq(Row(2 * 100))
)
+ val resultBeforeHandoff = sql("select * from streaming.stream_table_handoff order by id, name").collect()
sql("alter table streaming.stream_table_handoff compact 'streaming'")
Thread.sleep(5000)
+ val resultAfterHandoff = sql("select * from streaming.stream_table_handoff order by id, name").collect()
+ assertResult(resultBeforeHandoff)(resultAfterHandoff)
val newSegments = sql("show segments for table streaming.stream_table_handoff").collect()
assert(newSegments.length == 3 || newSegments.length == 5)
assertResult("Streaming")(newSegments((newSegments.length - 1) / 2).getString(1))
[20/50] [abbrv] carbondata git commit: [CARBONDATA-2096] Add query
test case for 'merge_small_files' distribution
Posted by ra...@apache.org.
[CARBONDATA-2096] Add query test case for 'merge_small_files' distribution
Add query test case for 'merge_small_files' distribution
This closes #1882
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d90280af
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d90280af
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d90280af
Branch: refs/heads/fgdatamap
Commit: d90280afc8adcab741c7aa29a99b450af78cd8e9
Parents: 24ba2fe
Author: QiangCai <qi...@qq.com>
Authored: Tue Jan 30 17:07:24 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 19:21:04 2018 +0800
----------------------------------------------------------------------
.../dataload/TestGlobalSortDataLoad.scala | 27 ++++++++++++++++++--
.../apache/spark/sql/test/util/QueryTest.scala | 1 +
2 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d90280af/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index 9ce9675..50a38f1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -25,14 +25,15 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.Row
+import org.apache.spark.sql.execution.BatchedDataSourceScanExec
import org.apache.spark.sql.test.TestQueryExecutor.projectPath
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.spark.rdd.CarbonScanRDD
class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
var filePath: String = s"$resourcesPath/globalsort"
@@ -272,7 +273,29 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbon_globalsort")
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
val segmentDir = carbonTablePath.getSegmentDir("0", "0")
- assertResult(5)(new File(segmentDir).listFiles().length)
+ assertResult(Math.max(4, defaultParallelism) + 1)(new File(segmentDir).listFiles().length)
+ }
+
+ test("Query with small files") {
+ try {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_MERGE_FILES)
+ for (i <- 0 until 10) {
+ sql(s"insert into carbon_globalsort select $i, 'name_$i', 'city_$i', ${ i % 100 }")
+ }
+ val df = sql("select * from carbon_globalsort")
+ val scanRdd = df.queryExecution.sparkPlan.collect {
+ case b: BatchedDataSourceScanExec if b.rdd.isInstanceOf[CarbonScanRDD] =>
+ b.rdd.asInstanceOf[CarbonScanRDD]
+ }.head
+ assertResult(defaultParallelism)(scanRdd.getPartitions.length)
+ assertResult(10)(df.count)
+ } finally {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT)
+ }
}
// ----------------------------------- INSERT INTO -----------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d90280af/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
index 0079d1e..b87473a 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
@@ -107,6 +107,7 @@ class QueryTest extends PlanTest {
val metastoredb = TestQueryExecutor.metastoredb
val integrationPath = TestQueryExecutor.integrationPath
val dblocation = TestQueryExecutor.location
+ val defaultParallelism = sqlContext.sparkContext.defaultParallelism
}
object QueryTest {
[41/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
deleted file mode 100644
index 6629d31..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-
-/**
- * This class is gives access to fixed length dimension data chunk store
- */
-public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk {
-
- /**
- * Constructor
- *
- * @param dataChunk data chunk
- * @param invertedIndex inverted index
- * @param invertedIndexReverse reverse inverted index
- * @param numberOfRows number of rows
- * @param columnValueSize size of each column value
- */
- public FixedLengthDimensionDataChunk(byte[] dataChunk, int[] invertedIndex,
- int[] invertedIndexReverse, int numberOfRows, int columnValueSize) {
- long totalSize = null != invertedIndex ?
- dataChunk.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) :
- dataChunk.length;
- dataChunkStore = DimensionChunkStoreFactory.INSTANCE
- .getDimensionChunkStore(columnValueSize, null != invertedIndex, numberOfRows, totalSize,
- DimensionStoreType.FIXEDLENGTH);
- dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunk);
- }
-
- /**
- * Below method will be used to fill the data based on offset and row id
- *
- * @param data data to filed
- * @param offset offset from which data need to be filed
- * @param index row id of the chunk
- * @param keyStructureInfo define the structure of the key
- * @return how many bytes was copied
- */
- @Override public int fillChunkData(byte[] data, int offset, int index,
- KeyStructureInfo keyStructureInfo) {
- dataChunkStore.fillRow(index, data, offset);
- return dataChunkStore.getColumnValueSize();
- }
-
- /**
- * Converts to column dictionary integer value
- *
- * @param rowId
- * @param columnIndex
- * @param row
- * @param restructuringInfo
- * @return
- */
- @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
- KeyStructureInfo restructuringInfo) {
- row[columnIndex] = dataChunkStore.getSurrogate(rowId);
- return columnIndex + 1;
- }
-
- /**
- * Fill the data to vector
- *
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = columnVectorInfo.size + offset;
- CarbonColumnVector vector = columnVectorInfo.vector;
- for (int j = offset; j < len; j++) {
- int dict = dataChunkStore.getSurrogate(j);
- if (columnVectorInfo.directDictionaryGenerator == null) {
- vector.putInt(vectorOffset++, dict);
- } else {
- Object valueFromSurrogate =
- columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict);
- if (valueFromSurrogate == null) {
- vector.putNull(vectorOffset++);
- } else {
- DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType();
- if (dataType == DataTypes.INT) {
- vector.putInt(vectorOffset++, (int) valueFromSurrogate);
- } else if (dataType == DataTypes.LONG) {
- vector.putLong(vectorOffset++, (long) valueFromSurrogate);
- } else {
- throw new IllegalArgumentException("unsupported data type: " +
- columnVectorInfo.directDictionaryGenerator.getReturnType());
- }
- }
- }
- }
- return column + 1;
- }
-
- /**
- * Fill the data to vector
- *
- * @param rowMapping
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo,
- int column, KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = columnVectorInfo.size + offset;
- CarbonColumnVector vector = columnVectorInfo.vector;
- for (int j = offset; j < len; j++) {
- int dict = dataChunkStore.getSurrogate(rowMapping[j]);
- if (columnVectorInfo.directDictionaryGenerator == null) {
- vector.putInt(vectorOffset++, dict);
- } else {
- Object valueFromSurrogate =
- columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict);
- if (valueFromSurrogate == null) {
- vector.putNull(vectorOffset++);
- } else {
- DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType();
- if (dataType == DataTypes.INT) {
- vector.putInt(vectorOffset++, (int) valueFromSurrogate);
- } else if (dataType == DataTypes.LONG) {
- vector.putLong(vectorOffset++, (long) valueFromSurrogate);
- } else {
- throw new IllegalArgumentException("unsupported data type: " +
- columnVectorInfo.directDictionaryGenerator.getReturnType());
- }
- }
- }
- }
- return column + 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
index fa0777b..5e8618b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -28,7 +28,7 @@ import org.apache.carbondata.core.memory.MemoryException;
/**
* Contains raw measure data
* 1. The read uncompressed raw data of column chunk with all pages is stored in this instance.
- * 2. The raw data can be converted to processed chunk using convertToColumnPage method
+ * 2. The raw data can be converted to processed chunk using decodeColumnPage method
* by specifying page number.
*/
public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
@@ -37,7 +37,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
private MeasureColumnChunkReader chunkReader;
- private FileHolder fileReader;
+ private FileReader fileReader;
public MeasureRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
MeasureColumnChunkReader chunkReader) {
@@ -48,14 +48,14 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
/**
* Convert all raw data with all pages to processed ColumnPage
*/
- public ColumnPage[] convertToColumnPage() {
+ public ColumnPage[] decodeAllColumnPages() {
if (columnPages == null) {
columnPages = new ColumnPage[pagesCount];
}
for (int i = 0; i < pagesCount; i++) {
try {
if (columnPages[i] == null) {
- columnPages[i] = chunkReader.convertToColumnPage(this, i);
+ columnPages[i] = chunkReader.decodeColumnPage(this, i);
}
} catch (Exception e) {
throw new RuntimeException(e);
@@ -68,21 +68,21 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
/**
* Convert raw data with specified `columnIndex` processed to ColumnPage
*/
- public ColumnPage convertToColumnPage(int columnIndex) {
- assert columnIndex < pagesCount;
+ public ColumnPage decodeColumnPage(int pageNumber) {
+ assert pageNumber < pagesCount;
if (columnPages == null) {
columnPages = new ColumnPage[pagesCount];
}
try {
- if (columnPages[columnIndex] == null) {
- columnPages[columnIndex] = chunkReader.convertToColumnPage(this, columnIndex);
+ if (columnPages[pageNumber] == null) {
+ columnPages[pageNumber] = chunkReader.decodeColumnPage(this, pageNumber);
}
} catch (IOException | MemoryException e) {
throw new RuntimeException(e);
}
- return columnPages[columnIndex];
+ return columnPages[pageNumber];
}
/**
@@ -95,7 +95,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
assert index < pagesCount;
try {
- return chunkReader.convertToColumnPage(this, index);
+ return chunkReader.decodeColumnPage(this, index);
} catch (IOException | MemoryException e) {
throw new RuntimeException(e);
}
@@ -111,11 +111,11 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
}
}
- public void setFileReader(FileHolder fileReader) {
+ public void setFileReader(FileReader fileReader) {
this.fileReader = fileReader;
}
- public FileHolder getFileReader() {
+ public FileReader getFileReader() {
return fileReader;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
new file mode 100644
index 0000000..d03b2de
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.impl;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * This class is gives access to variable length dimension data chunk store
+ */
+public class VariableLengthDimensionColumnPage extends AbstractDimensionColumnPage {
+
+ /**
+ * Constructor for this class
+ * @param dataChunks
+ * @param invertedIndex
+ * @param invertedIndexReverse
+ * @param numberOfRows
+ */
+ public VariableLengthDimensionColumnPage(byte[] dataChunks, int[] invertedIndex,
+ int[] invertedIndexReverse, int numberOfRows) {
+ long totalSize = null != invertedIndex ?
+ (dataChunks.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) + (
+ numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)) :
+ (dataChunks.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ dataChunkStore = DimensionChunkStoreFactory.INSTANCE
+ .getDimensionChunkStore(0, null != invertedIndex, numberOfRows, totalSize,
+ DimensionStoreType.VARIABLELENGTH);
+ dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunks);
+ }
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param rowId row id of the chunk
+ * @param offset offset from which data need to be filed
+ * @param data data to filed
+ * @param restructuringInfo define the structure of the key
+ * @return how many bytes was copied
+ */
+ @Override public int fillRawData(int rowId, int offset, byte[] data,
+ KeyStructureInfo restructuringInfo) {
+ // no required in this case because this column chunk is not the part if
+ // mdkey
+ return 0;
+ }
+
+ /**
+ * Converts to column dictionary integer value
+ *
+ * @param rowId
+ * @param chunkIndex
+ * @param outputSurrogateKey
+ * @param restructuringInfo
+ * @return
+ */
+ @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
+ KeyStructureInfo restructuringInfo) {
+ return chunkIndex + 1;
+ }
+
+ /**
+ * @return whether column is dictionary column or not
+ */
+ @Override public boolean isNoDicitionaryColumn() {
+ return true;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ for (int i = offset; i < len; i++) {
+ // Considering only String case now as we support only
+ // string in no dictionary case at present.
+ dataChunkStore.fillRow(i, vector, vectorOffset++);
+ }
+ return chunkIndex + 1;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param filteredRowId
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo,
+ int chunkIndex, KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ for (int i = offset; i < len; i++) {
+ // Considering only String case now as we support only
+ // string in no dictionary case at present.
+ dataChunkStore.fillRow(filteredRowId[i], vector, vectorOffset++);
+ }
+ return chunkIndex + 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
deleted file mode 100644
index 6c47bf5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-
-/**
- * This class is gives access to variable length dimension data chunk store
- */
-public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk {
-
- /**
- * Constructor for this class
- * @param dataChunks
- * @param invertedIndex
- * @param invertedIndexReverse
- * @param numberOfRows
- */
- public VariableLengthDimensionDataChunk(byte[] dataChunks, int[] invertedIndex,
- int[] invertedIndexReverse, int numberOfRows) {
- long totalSize = null != invertedIndex ?
- (dataChunks.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) + (
- numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)) :
- (dataChunks.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE));
- dataChunkStore = DimensionChunkStoreFactory.INSTANCE
- .getDimensionChunkStore(0, null != invertedIndex, numberOfRows, totalSize,
- DimensionStoreType.VARIABLELENGTH);
- dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunks);
- }
-
- /**
- * Below method will be used to fill the data based on offset and row id
- *
- * @param data data to filed
- * @param offset offset from which data need to be filed
- * @param index row id of the chunk
- * @param restructuringInfo define the structure of the key
- * @return how many bytes was copied
- */
- @Override public int fillChunkData(byte[] data, int offset, int index,
- KeyStructureInfo restructuringInfo) {
- // no required in this case because this column chunk is not the part if
- // mdkey
- return 0;
- }
-
- /**
- * Converts to column dictionary integer value
- *
- * @param rowId
- * @param columnIndex
- * @param row
- * @param restructuringInfo
- * @return
- */
- @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
- KeyStructureInfo restructuringInfo) {
- return columnIndex + 1;
- }
-
- /**
- * @return whether column is dictionary column or not
- */
- @Override public boolean isNoDicitionaryColumn() {
- return true;
- }
-
- /**
- * @return length of each column
- */
- @Override public int getColumnValueSize() {
- return -1;
- }
-
- /**
- * Fill the data to vector
- *
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- CarbonColumnVector vector = columnVectorInfo.vector;
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- for (int i = offset; i < len; i++) {
- // Considering only String case now as we support only
- // string in no dictionary case at present.
- dataChunkStore.fillRow(i, vector, vectorOffset++);
- }
- return column + 1;
- }
-
- /**
- * Fill the data to vector
- *
- * @param rowMapping
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo,
- int column, KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- CarbonColumnVector vector = columnVectorInfo.vector;
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- for (int i = offset; i < len; i++) {
- // Considering only String case now as we support only
- // string in no dictionary case at present.
- dataChunkStore.fillRow(rowMapping[i], vector, vectorOffset++);
- }
- return column + 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
index 7b5b9c8..fd81973 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -18,8 +18,8 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.memory.MemoryException;
@@ -34,20 +34,20 @@ public interface DimensionColumnChunkReader {
* Below method will be used to read the chunk based on block indexes
*
* @param fileReader file reader to read the blocks from file
- * @param blockletIndexes blocklets to be read
+ * @param columnIndexRange blocklets to be read
* @return dimension column chunks
*/
- DimensionRawColumnChunk[] readRawDimensionChunks(FileHolder fileReader, int[][] blockletIndexes)
+ DimensionRawColumnChunk[] readRawDimensionChunks(FileReader fileReader, int[][] columnIndexRange)
throws IOException;
/**
* Below method will be used to read the chunk based on block index
*
* @param fileReader file reader to read the blocks from file
- * @param blockletIndex block to be read
+ * @param columnIndex column to be read
* @return dimension column chunk
*/
- DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, int blockletIndex)
+ DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int columnIndex)
throws IOException;
/**
@@ -58,6 +58,6 @@ public interface DimensionColumnChunkReader {
* @return
* @throws IOException
*/
- DimensionColumnDataChunk convertToDimensionChunk(DimensionRawColumnChunk dimensionRawColumnChunk,
+ DimensionColumnPage decodeColumnPage(DimensionRawColumnChunk dimensionRawColumnChunk,
int pageNumber) throws IOException, MemoryException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
index 02dc6a2..bf76025 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -18,7 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.memory.MemoryException;
@@ -32,20 +32,20 @@ public interface MeasureColumnChunkReader {
* Method to read the blocks data based on block indexes
*
* @param fileReader file reader to read the blocks
- * @param blockIndexes blocks to be read
+ * @param columnIndexRange blocks to be read
* @return measure data chunks
*/
- MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
+ MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader, int[][] columnIndexRange)
throws IOException;
/**
* Method to read the blocks data based on block index
*
* @param fileReader file reader to read the blocks
- * @param blockIndex block to be read
+ * @param columnIndex block to be read
* @return measure data chunk
*/
- MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int blockIndex)
+ MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException;
/**
@@ -55,7 +55,7 @@ public interface MeasureColumnChunkReader {
* @return
* @throws IOException
*/
- ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
+ ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
int pageNumber) throws IOException, MemoryException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
index f083612..60950c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.format.Encoding;
@@ -55,43 +55,43 @@ public abstract class AbstractChunkReaderV2V3Format extends AbstractChunkReader
* For last column read is separately and process
*
* @param fileReader file reader to read the blocks from file
- * @param blockletIndexes blocks range to be read
+ * @param columnIndexRange column index range to be read
* @return dimension column chunks
*/
- @Override public DimensionRawColumnChunk[] readRawDimensionChunks(final FileHolder fileReader,
- final int[][] blockletIndexes) throws IOException {
+ @Override public DimensionRawColumnChunk[] readRawDimensionChunks(final FileReader fileReader,
+ final int[][] columnIndexRange) throws IOException {
// read the column chunk based on block index and add
DimensionRawColumnChunk[] dataChunks =
new DimensionRawColumnChunk[dimensionChunksOffset.size()];
// if blocklet index is empty then return empry data chunk
- if (blockletIndexes.length == 0) {
+ if (columnIndexRange.length == 0) {
return dataChunks;
}
DimensionRawColumnChunk[] groupChunk = null;
int index = 0;
// iterate till block indexes -1 as block index will be in sorted order, so to avoid
// the last column reading in group
- for (int i = 0; i < blockletIndexes.length - 1; i++) {
+ for (int i = 0; i < columnIndexRange.length - 1; i++) {
index = 0;
groupChunk =
- readRawDimensionChunksInGroup(fileReader, blockletIndexes[i][0], blockletIndexes[i][1]);
- for (int j = blockletIndexes[i][0]; j <= blockletIndexes[i][1]; j++) {
+ readRawDimensionChunksInGroup(fileReader, columnIndexRange[i][0], columnIndexRange[i][1]);
+ for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
dataChunks[j] = groupChunk[index++];
}
}
// check last index is present in block index, if it is present then read separately
- if (blockletIndexes[blockletIndexes.length - 1][0] == dimensionChunksOffset.size() - 1) {
- dataChunks[blockletIndexes[blockletIndexes.length - 1][0]] =
- readRawDimensionChunk(fileReader, blockletIndexes[blockletIndexes.length - 1][0]);
+ if (columnIndexRange[columnIndexRange.length - 1][0] == dimensionChunksOffset.size() - 1) {
+ dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] =
+ readRawDimensionChunk(fileReader, columnIndexRange[columnIndexRange.length - 1][0]);
}
// otherwise read the data in group
else {
- groupChunk =
- readRawDimensionChunksInGroup(fileReader, blockletIndexes[blockletIndexes.length - 1][0],
- blockletIndexes[blockletIndexes.length - 1][1]);
+ groupChunk = readRawDimensionChunksInGroup(
+ fileReader, columnIndexRange[columnIndexRange.length - 1][0],
+ columnIndexRange[columnIndexRange.length - 1][1]);
index = 0;
- for (int j = blockletIndexes[blockletIndexes.length - 1][0];
- j <= blockletIndexes[blockletIndexes.length - 1][1]; j++) {
+ for (int j = columnIndexRange[columnIndexRange.length - 1][0];
+ j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) {
dataChunks[j] = groupChunk[index++];
}
}
@@ -109,7 +109,7 @@ public abstract class AbstractChunkReaderV2V3Format extends AbstractChunkReader
* @return measure raw chunkArray
* @throws IOException
*/
- protected abstract DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ protected abstract DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader,
int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index 27a4d89..0dc1c1b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -20,12 +20,12 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -61,14 +61,14 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
* Below method will be used to read the raw chunk based on block indexes
*
* @param fileReader file reader to read the blocks from file
- * @param blockletIndexes blocks to be read
+ * @param columnIndexRange blocks to be read
* @return dimension column chunks
*/
- @Override public DimensionRawColumnChunk[] readRawDimensionChunks(FileHolder fileReader,
- int[][] blockletIndexes) throws IOException {
+ @Override public DimensionRawColumnChunk[] readRawDimensionChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
DimensionRawColumnChunk[] dataChunks = new DimensionRawColumnChunk[dimensionColumnChunk.size()];
- for (int i = 0; i < blockletIndexes.length; i++) {
- for (int j = blockletIndexes[i][0]; j <= blockletIndexes[i][1]; j++) {
+ for (int i = 0; i < columnIndexRange.length; i++) {
+ for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
dataChunks[j] = readRawDimensionChunk(fileReader, j);
}
}
@@ -82,7 +82,7 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
* @param columnIndex column to be read
* @return dimension column chunk
*/
- @Override public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader,
+ @Override public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader,
int columnIndex) throws IOException {
DataChunk dataChunk = dimensionColumnChunk.get(columnIndex);
ByteBuffer buffer = null;
@@ -92,20 +92,20 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
}
DimensionRawColumnChunk rawColumnChunk = new DimensionRawColumnChunk(columnIndex, buffer, 0,
dataChunk.getDataPageLength(), this);
- rawColumnChunk.setFileHolder(fileReader);
+ rawColumnChunk.setFileReader(fileReader);
rawColumnChunk.setPagesCount(1);
rawColumnChunk.setRowCount(new int[] { numberOfRows });
return rawColumnChunk;
}
- @Override public DimensionColumnDataChunk convertToDimensionChunk(
+ @Override public DimensionColumnPage decodeColumnPage(
DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException {
int blockIndex = dimensionRawColumnChunk.getColumnIndex();
byte[] dataPage = null;
int[] invertedIndexes = null;
int[] invertedIndexesReverse = null;
int[] rlePage = null;
- FileHolder fileReader = dimensionRawColumnChunk.getFileReader();
+ FileReader fileReader = dimensionRawColumnChunk.getFileReader();
ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
dataPage = COMPRESSOR.unCompressByte(rawData.array(), (int) dimensionRawColumnChunk.getOffSet(),
@@ -145,23 +145,23 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
rlePage = null;
}
// fill chunk attributes
- DimensionColumnDataChunk columnDataChunk = null;
+ DimensionColumnPage columnDataChunk = null;
if (dataChunk.isRowMajor()) {
// to store fixed length column chunk values
- columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, eachColumnValueSize[blockIndex],
- numberOfRows);
+ columnDataChunk = new ColumnGroupDimensionColumnPage(
+ dataPage, eachColumnValueSize[blockIndex], numberOfRows);
}
// if no dictionary column then first create a no dictionary column chunk
// and set to data chunk instance
else if (!CarbonUtil
.hasEncoding(dataChunk.getEncodingList(), Encoding.DICTIONARY)) {
columnDataChunk =
- new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
numberOfRows);
} else {
// to store fixed length column chunk values
columnDataChunk =
- new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
numberOfRows, eachColumnValueSize[blockIndex]);
}
return columnDataChunk;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index b43f89c..31fa819 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -19,12 +19,12 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -56,7 +56,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
* @param columnIndex column to be read
* @return dimension column chunk
*/
- public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, int columnIndex)
+ public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int columnIndex)
throws IOException {
int length = 0;
if (dimensionChunksOffset.size() - 1 == columnIndex) {
@@ -73,7 +73,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
}
DimensionRawColumnChunk rawColumnChunk =
new DimensionRawColumnChunk(columnIndex, buffer, 0, length, this);
- rawColumnChunk.setFileHolder(fileReader);
+ rawColumnChunk.setFileReader(fileReader);
rawColumnChunk.setPagesCount(1);
rawColumnChunk.setRowCount(new int[] { numberOfRows });
return rawColumnChunk;
@@ -90,7 +90,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
* @return measure raw chunkArray
* @throws IOException
*/
- protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader,
int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
long currentDimensionOffset = dimensionChunksOffset.get(startColumnBlockletIndex);
ByteBuffer buffer = null;
@@ -106,7 +106,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
int currentLength = (int) (dimensionChunksOffset.get(i + 1) - dimensionChunksOffset.get(i));
dataChunks[index] =
new DimensionRawColumnChunk(i, buffer, runningLength, currentLength, this);
- dataChunks[index].setFileHolder(fileReader);
+ dataChunks[index].setFileReader(fileReader);
dataChunks[index].setPagesCount(1);
dataChunks[index].setRowCount(new int[] { numberOfRows });
runningLength += currentLength;
@@ -115,7 +115,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
return dataChunks;
}
- public DimensionColumnDataChunk convertToDimensionChunk(
+ public DimensionColumnPage decodeColumnPage(
DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException {
byte[] dataPage = null;
int[] invertedIndexes = null;
@@ -169,23 +169,23 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
dataPage = UnBlockIndexer.uncompressData(dataPage, rlePage, eachColumnValueSize[blockIndex]);
}
// fill chunk attributes
- DimensionColumnDataChunk columnDataChunk = null;
+ DimensionColumnPage columnDataChunk = null;
if (dimensionColumnChunk.isRowMajor()) {
// to store fixed length column chunk values
- columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, eachColumnValueSize[blockIndex],
- numberOfRows);
+ columnDataChunk = new ColumnGroupDimensionColumnPage(
+ dataPage, eachColumnValueSize[blockIndex], numberOfRows);
}
// if no dictionary column then first create a no dictionary column chunk
// and set to data chunk instance
else if (!hasEncoding(dimensionColumnChunk.encoders, Encoding.DICTIONARY)) {
columnDataChunk =
- new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
numberOfRows);
} else {
// to store fixed length column chunk values
columnDataChunk =
- new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
numberOfRows, eachColumnValueSize[blockIndex]);
}
return columnDataChunk;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
index 1edfd09..60f0b67 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
@@ -20,8 +20,8 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -70,7 +70,8 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3
* @param blockletColumnIndex blocklet index of the column in carbon data file
* @return dimension raw chunk
*/
- public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader,
+ @Override
+ public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader,
int blockletColumnIndex) throws IOException {
// get the current dimension offset
long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex);
@@ -116,7 +117,7 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3
* @param endBlockletColumnIndex blocklet index of the last dimension column
* @ DimensionRawColumnChunk array
*/
- protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader,
int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException {
// create raw chunk for each dimension column
DimensionRawColumnChunk[] dimensionDataChunks =
@@ -136,7 +137,7 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3
* @param pageNumber number
* @return DimensionColumnDataChunk
*/
- @Override public DimensionColumnDataChunk convertToDimensionChunk(
+ @Override public DimensionColumnPage decodeColumnPage(
DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber)
throws IOException, MemoryException {
// data chunk of page
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
index 566e9b7..0fdc515 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
@@ -20,11 +20,11 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format;
import org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
@@ -78,23 +78,23 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
* 5. Create the raw chunk object and fill the details
*
* @param fileReader reader for reading the column from carbon data file
- * @param blockletColumnIndex blocklet index of the column in carbon data file
+ * @param columnIndex blocklet index of the column in carbon data file
* @return dimension raw chunk
*/
- public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader,
- int blockletColumnIndex) throws IOException {
+ public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader,
+ int columnIndex) throws IOException {
// get the current dimension offset
- long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex);
+ long currentDimensionOffset = dimensionChunksOffset.get(columnIndex);
int length = 0;
// to calculate the length of the data to be read
// column other than last column we can subtract the offset of current column with
// next column and get the total length.
// but for last column we need to use lastDimensionOffset which is the end position
// of the last dimension, we can subtract current dimension offset from lastDimesionOffset
- if (dimensionChunksOffset.size() - 1 == blockletColumnIndex) {
+ if (dimensionChunksOffset.size() - 1 == columnIndex) {
length = (int) (lastDimensionOffsets - currentDimensionOffset);
} else {
- length = (int) (dimensionChunksOffset.get(blockletColumnIndex + 1) - currentDimensionOffset);
+ length = (int) (dimensionChunksOffset.get(columnIndex + 1) - currentDimensionOffset);
}
ByteBuffer buffer = null;
// read the data from carbon data file
@@ -103,15 +103,15 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
}
// get the data chunk which will have all the details about the data pages
DataChunk3 dataChunk = CarbonUtil.readDataChunk3(buffer, 0, length);
- return getDimensionRawColumnChunk(fileReader, blockletColumnIndex, 0, length, buffer,
+ return getDimensionRawColumnChunk(fileReader, columnIndex, 0, length, buffer,
dataChunk);
}
- protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileHolder fileReader,
- int blockletColumnIndex, long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) {
+ protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileReader fileReader,
+ int columnIndex, long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) {
// creating a raw chunks instance and filling all the details
DimensionRawColumnChunk rawColumnChunk =
- new DimensionRawColumnChunk(blockletColumnIndex, buffer, offset, length, this);
+ new DimensionRawColumnChunk(columnIndex, buffer, offset, length, this);
int numberOfPages = dataChunk.getPage_length().size();
byte[][] maxValueOfEachPage = new byte[numberOfPages][];
byte[][] minValueOfEachPage = new byte[numberOfPages][];
@@ -124,7 +124,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
eachPageLength[i] = dataChunk.getData_chunk_list().get(i).getNumberOfRowsInpage();
}
rawColumnChunk.setDataChunkV3(dataChunk);
- rawColumnChunk.setFileHolder(fileReader);
+ rawColumnChunk.setFileReader(fileReader);
rawColumnChunk.setPagesCount(dataChunk.getPage_length().size());
rawColumnChunk.setMaxValues(maxValueOfEachPage);
rawColumnChunk.setMinValues(minValueOfEachPage);
@@ -153,7 +153,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
* blocklet index of the last dimension column
* @ DimensionRawColumnChunk array
*/
- protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader,
int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException {
// to calculate the length of the data to be read
// column we can subtract the offset of start column offset with
@@ -188,9 +188,9 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
*
* @param rawColumnPage dimension raw chunk
* @param pageNumber number
- * @return DimensionColumnDataChunk
+ * @return DimensionColumnPage
*/
- @Override public DimensionColumnDataChunk convertToDimensionChunk(
+ @Override public DimensionColumnPage decodeColumnPage(
DimensionRawColumnChunk rawColumnPage, int pageNumber) throws IOException, MemoryException {
// data chunk of blocklet column
DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3();
@@ -228,20 +228,19 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
return false;
}
- protected DimensionColumnDataChunk decodeDimension(DimensionRawColumnChunk rawColumnPage,
+ protected DimensionColumnPage decodeDimension(DimensionRawColumnChunk rawColumnPage,
ByteBuffer pageData, DataChunk2 pageMetadata, int offset)
throws IOException, MemoryException {
if (isEncodedWithMeta(pageMetadata)) {
ColumnPage decodedPage = decodeDimensionByMeta(pageMetadata, pageData, offset);
- return new ColumnPageWrapper(decodedPage,
- eachColumnValueSize[rawColumnPage.getColumnIndex()]);
+ return new ColumnPageWrapper(decodedPage);
} else {
// following code is for backward compatibility
return decodeDimensionLegacy(rawColumnPage, pageData, pageMetadata, offset);
}
}
- private DimensionColumnDataChunk decodeDimensionLegacy(DimensionRawColumnChunk rawColumnPage,
+ private DimensionColumnPage decodeDimensionLegacy(DimensionRawColumnChunk rawColumnPage,
ByteBuffer pageData, DataChunk2 pageMetadata, int offset) {
byte[] dataPage;
int[] rlePage;
@@ -267,18 +266,18 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
eachColumnValueSize[rawColumnPage.getColumnIndex()]);
}
- DimensionColumnDataChunk columnDataChunk = null;
+ DimensionColumnPage columnDataChunk = null;
// if no dictionary column then first create a no dictionary column chunk
// and set to data chunk instance
if (!hasEncoding(pageMetadata.encoders, Encoding.DICTIONARY)) {
columnDataChunk =
- new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
pageMetadata.getNumberOfRowsInpage());
} else {
// to store fixed length column chunk values
columnDataChunk =
- new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
pageMetadata.getNumberOfRowsInpage(),
eachColumnValueSize[rawColumnPage.getColumnIndex()]);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
index 2239a2b..a3ed339 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
@@ -56,36 +56,40 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu
* separately and process
*
* @param fileReader file reader to read the blocks from file
- * @param blockIndexes blocks range to be read
+ * @param columnIndexRange blocks range to be read, columnIndexGroup[i] is one group, inside the
+ * group, columnIndexGroup[i][0] is start column index,
+ * and columnIndexGroup[i][1] is end column index
* @return measure column chunks
* @throws IOException
*/
- public MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
- throws IOException {
+ public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
// read the column chunk based on block index and add
MeasureRawColumnChunk[] dataChunks =
new MeasureRawColumnChunk[measureColumnChunkOffsets.size()];
- if (blockIndexes.length == 0) {
+ if (columnIndexRange.length == 0) {
return dataChunks;
}
MeasureRawColumnChunk[] groupChunk = null;
int index = 0;
- for (int i = 0; i < blockIndexes.length - 1; i++) {
+ for (int i = 0; i < columnIndexRange.length - 1; i++) {
index = 0;
- groupChunk = readRawMeasureChunksInGroup(fileReader, blockIndexes[i][0], blockIndexes[i][1]);
- for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) {
+ groupChunk = readRawMeasureChunksInGroup(
+ fileReader, columnIndexRange[i][0], columnIndexRange[i][1]);
+ for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
dataChunks[j] = groupChunk[index++];
}
}
- if (blockIndexes[blockIndexes.length - 1][0] == measureColumnChunkOffsets.size() - 1) {
- dataChunks[blockIndexes[blockIndexes.length - 1][0]] =
- readRawMeasureChunk(fileReader, blockIndexes[blockIndexes.length - 1][0]);
+ if (columnIndexRange[columnIndexRange.length - 1][0] == measureColumnChunkOffsets.size() - 1) {
+ dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] =
+ readRawMeasureChunk(fileReader, columnIndexRange[columnIndexRange.length - 1][0]);
} else {
- groupChunk = readRawMeasureChunksInGroup(fileReader, blockIndexes[blockIndexes.length - 1][0],
- blockIndexes[blockIndexes.length - 1][1]);
+ groupChunk = readRawMeasureChunksInGroup(
+ fileReader, columnIndexRange[columnIndexRange.length - 1][0],
+ columnIndexRange[columnIndexRange.length - 1][1]);
index = 0;
- for (int j = blockIndexes[blockIndexes.length - 1][0];
- j <= blockIndexes[blockIndexes.length - 1][1]; j++) {
+ for (int j = columnIndexRange[columnIndexRange.length - 1][0];
+ j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) {
dataChunks[j] = groupChunk[index++];
}
}
@@ -112,12 +116,12 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu
* data from
*
* @param fileReader file reader to read the data
- * @param startColumnBlockletIndex first column blocklet index to be read
- * @param endColumnBlockletIndex end column blocklet index to be read
+ * @param startColumnIndex first column index to be read
+ * @param endColumnIndex end column index to be read
* @return measure raw chunkArray
* @throws IOException
*/
- protected abstract MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
- int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException;
+ protected abstract MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader,
+ int startColumnIndex, int endColumnIndex) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index ae55375..f0c1b75 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -56,14 +56,14 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
* Method to read the blocks data based on block indexes
*
* @param fileReader file reader to read the blocks
- * @param blockIndexes blocks to be read
+ * @param columnIndexRange blocks to be read
* @return measure data chunks
*/
- @Override public MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
MeasureRawColumnChunk[] datChunk = new MeasureRawColumnChunk[measureColumnChunks.size()];
- for (int i = 0; i < blockIndexes.length; i++) {
- for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) {
+ for (int i = 0; i < columnIndexRange.length; i++) {
+ for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
datChunk[j] = readRawMeasureChunk(fileReader, j);
}
}
@@ -77,7 +77,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
* @param columnIndex column to be read
* @return measure data chunk
*/
- @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int columnIndex)
+ @Override public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException {
DataChunk dataChunk = measureColumnChunks.get(columnIndex);
ByteBuffer buffer = fileReader
@@ -91,7 +91,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
}
@Override
- public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
+ public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
int pageNumber) throws IOException, MemoryException {
int blockIndex = measureRawColumnChunk.getColumnIndex();
DataChunk dataChunk = measureColumnChunks.get(blockIndex);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 2ddc202..e990b4b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -48,7 +48,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
}
@Override
- public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int columnIndex)
+ public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException {
int dataLength = 0;
if (measureColumnChunkOffsets.size() - 1 == columnIndex) {
@@ -76,30 +76,30 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
* data from
*
* @param fileReader file reader to read the data
- * @param startColumnBlockletIndex first column blocklet index to be read
- * @param endColumnBlockletIndex end column blocklet index to be read
+ * @param startColumnIndex first column blocklet index to be read
+ * @param endColumnIndex end column blocklet index to be read
* @return measure raw chunkArray
* @throws IOException
*/
- protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
- int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
- long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnBlockletIndex);
+ protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader,
+ int startColumnIndex, int endColumnIndex) throws IOException {
+ long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex);
ByteBuffer buffer = null;
synchronized (fileReader) {
buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset,
- (int) (measureColumnChunkOffsets.get(endColumnBlockletIndex + 1) - currentMeasureOffset));
+ (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset));
}
MeasureRawColumnChunk[] dataChunks =
- new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1];
+ new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1];
int runningLength = 0;
int index = 0;
- for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
+ for (int i = startColumnIndex; i <= endColumnIndex; i++) {
int currentLength =
(int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i));
MeasureRawColumnChunk measureRawColumnChunk =
new MeasureRawColumnChunk(i, buffer, runningLength, currentLength, this);
- measureRawColumnChunk.setFileReader(fileReader);
measureRawColumnChunk.setRowCount(new int[] { numberOfRows });
+ measureRawColumnChunk.setFileReader(fileReader);
measureRawColumnChunk.setPagesCount(1);
dataChunks[index] = measureRawColumnChunk;
runningLength += currentLength;
@@ -108,7 +108,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
return dataChunks;
}
- public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
+ public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
int pageNumber) throws IOException, MemoryException {
int copyPoint = (int) measureRawColumnChunk.getOffSet();
int blockIndex = measureRawColumnChunk.getColumnIndex();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
index 4f6987b..6dc02a3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -70,7 +70,7 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
* @param columnIndex column to be read
* @return measure raw chunk
*/
- @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader,
+ @Override public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader,
int columnIndex) throws IOException {
int dataLength = 0;
// to calculate the length of the data to be read
@@ -99,9 +99,8 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
dataChunk);
}
- protected MeasureRawColumnChunk getMeasureRawColumnChunk(FileHolder fileReader,
- int columnIndex, long offset, int dataLength, ByteBuffer buffer,
- DataChunk3 dataChunk) {
+ MeasureRawColumnChunk getMeasureRawColumnChunk(FileReader fileReader, int columnIndex,
+ long offset, int dataLength, ByteBuffer buffer, DataChunk3 dataChunk) {
// creating a raw chunks instance and filling all the details
MeasureRawColumnChunk rawColumnChunk =
new MeasureRawColumnChunk(columnIndex, buffer, offset, dataLength, this);
@@ -140,30 +139,30 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
*
* @param fileReader
* reader which will be used to read the measure columns data from file
- * @param startColumnBlockletIndex
- * blocklet index of the first measure column
- * @param endColumnBlockletIndex
- * blocklet index of the last measure column
+ * @param startColumnIndex
+ * column index of the first measure column
+ * @param endColumnIndex
+ * column index of the last measure column
* @return MeasureRawColumnChunk array
*/
- protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
- int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
+ protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader,
+ int startColumnIndex, int endColumnIndex) throws IOException {
// to calculate the length of the data to be read
// column we can subtract the offset of start column offset with
// end column+1 offset and get the total length.
- long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnBlockletIndex);
+ long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex);
ByteBuffer buffer = null;
// read the data from carbon data file
synchronized (fileReader) {
buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset,
- (int) (measureColumnChunkOffsets.get(endColumnBlockletIndex + 1) - currentMeasureOffset));
+ (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset));
}
// create raw chunk for each measure column
MeasureRawColumnChunk[] measureDataChunk =
- new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1];
+ new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1];
int runningLength = 0;
int index = 0;
- for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
+ for (int i = startColumnIndex; i <= endColumnIndex; i++) {
int currentLength =
(int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i));
DataChunk3 dataChunk =
@@ -180,25 +179,25 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
/**
* Below method will be used to convert the compressed measure chunk raw data to actual data
*
- * @param rawColumnPage measure raw chunk
+ * @param rawColumnChunk measure raw chunk
* @param pageNumber number
- * @return DimensionColumnDataChunk
+ * @return DimensionColumnPage
*/
@Override
- public ColumnPage convertToColumnPage(
- MeasureRawColumnChunk rawColumnPage, int pageNumber)
+ public ColumnPage decodeColumnPage(
+ MeasureRawColumnChunk rawColumnChunk, int pageNumber)
throws IOException, MemoryException {
// data chunk of blocklet column
- DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3();
+ DataChunk3 dataChunk3 = rawColumnChunk.getDataChunkV3();
// data chunk of page
DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber);
// calculating the start point of data
// as buffer can contain multiple column data, start point will be datachunkoffset +
// data chunk length + page offset
- int offset = (int) rawColumnPage.getOffSet() +
- measureColumnChunkLength.get(rawColumnPage.getColumnIndex()) +
+ int offset = (int) rawColumnChunk.getOffSet() +
+ measureColumnChunkLength.get(rawColumnChunk.getColumnIndex()) +
dataChunk3.getPage_offset().get(pageNumber);
- ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnPage.getRawData(), offset);
+ ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnChunk.getRawData(), offset);
decodedPage.setNullBits(getNullBitSet(pageMetadata.presence));
return decodedPage;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
index 31ff4c0..6b37575 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
@@ -20,7 +20,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.memory.MemoryException;
@@ -66,7 +66,7 @@ public class CompressedMsrChunkFileBasedPageLevelReaderV3
* @param blockletColumnIndex blocklet index of the column in carbon data file
* @return measure raw chunk
*/
- @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader,
+ @Override public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader,
int blockletColumnIndex) throws IOException {
int dataLength = 0;
// to calculate the length of the data to be read
@@ -110,7 +110,7 @@ public class CompressedMsrChunkFileBasedPageLevelReaderV3
* @param endColumnBlockletIndex blocklet index of the last measure column
* @return MeasureRawColumnChunk array
*/
- protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
+ protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader,
int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
// create raw chunk for each measure column
MeasureRawColumnChunk[] measureDataChunk =
@@ -130,7 +130,7 @@ public class CompressedMsrChunkFileBasedPageLevelReaderV3
* @param pageNumber number
* @return DimensionColumnDataChunk
*/
- @Override public ColumnPage convertToColumnPage(
+ @Override public ColumnPage decodeColumnPage(
MeasureRawColumnChunk rawColumnPage, int pageNumber)
throws IOException, MemoryException {
// data chunk of blocklet column
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
index fbdb499..c89ecc3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
@@ -17,48 +17,45 @@
package org.apache.carbondata.core.datastore.chunk.store;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-public class ColumnPageWrapper implements DimensionColumnDataChunk {
+public class ColumnPageWrapper implements DimensionColumnPage {
private ColumnPage columnPage;
- private int columnValueSize;
- public ColumnPageWrapper(ColumnPage columnPage, int columnValueSize) {
+ public ColumnPageWrapper(ColumnPage columnPage) {
this.columnPage = columnPage;
- this.columnValueSize = columnValueSize;
}
@Override
- public int fillChunkData(byte[] data, int offset, int columnIndex,
- KeyStructureInfo restructuringInfo) {
+ public int fillRawData(int rowId, int offset, byte[] data, KeyStructureInfo restructuringInfo) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
KeyStructureInfo restructuringInfo) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
+ public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
KeyStructureInfo restructuringInfo) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, int column,
+ public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int chunkIndex,
KeyStructureInfo restructuringInfo) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public byte[] getChunkData(int columnIndex) {
- return columnPage.getBytes(columnIndex);
+ public byte[] getChunkData(int rowId) {
+ return columnPage.getBytes(rowId);
}
@Override
@@ -66,7 +63,7 @@ public class ColumnPageWrapper implements DimensionColumnDataChunk {
throw new UnsupportedOperationException("internal error");
}
- @Override public int getInvertedReverseIndex(int invertedIndex) {
+ @Override public int getInvertedReverseIndex(int rowId) {
throw new UnsupportedOperationException("internal error");
}
@@ -76,17 +73,12 @@ public class ColumnPageWrapper implements DimensionColumnDataChunk {
}
@Override
- public int getColumnValueSize() {
- return columnValueSize;
- }
-
- @Override
public boolean isExplicitSorted() {
return false;
}
@Override
- public int compareTo(int index, byte[] compareValue) {
+ public int compareTo(int rowId, byte[] compareValue) {
throw new UnsupportedOperationException("internal error");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
index 5072c75..28aed5b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
@@ -69,10 +69,10 @@ public interface DimensionDataChunkStore {
/**
* Below method will be used to get the reverse Inverted Index
- * @param invertedIndex
+ * @param rowId
* @return reverse Inverted Index
*/
- int getInvertedReverseIndex(int invertedIndex);
+ int getInvertedReverseIndex(int rowId);
/**
* Below method will be used to get the surrogate key of the
@@ -102,9 +102,9 @@ public interface DimensionDataChunkStore {
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- int compareTo(int index, byte[] compareValue);
+ int compareTo(int rowId, byte[] compareValue);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
index 8656878..41218d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
@@ -97,13 +97,13 @@ public class SafeFixedLengthDimensionDataChunkStore extends SafeAbsractDimension
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
return ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(data, index * columnValueSize, columnValueSize, compareValue, 0,
+ .compareTo(data, rowId * columnValueSize, columnValueSize, compareValue, 0,
columnValueSize);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
index db83198..f498c6e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -162,7 +162,7 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
}
}
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// now to get the row from memory block we need to do following thing
// 1. first get the current offset
// 2. if it's not a last row- get the next row offset
@@ -171,11 +171,11 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
// length
// get the offset of set of data
- int currentDataOffset = dataOffsets[index];
+ int currentDataOffset = dataOffsets[rowId];
short length = 0;
// calculating the length of data
- if (index < numberOfRows - 1) {
- length = (short) (dataOffsets[index + 1] - (currentDataOffset
+ if (rowId < numberOfRows - 1) {
+ length = (short) (dataOffsets[rowId + 1] - (currentDataOffset
+ CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
} else {
// for last record
[19/50] [abbrv] carbondata git commit: [HOTFIX] Correct CI url and
add standard partition usage
Posted by ra...@apache.org.
[HOTFIX] Correct CI url and add standard partition usage
This closes #1889
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/24ba2fe2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/24ba2fe2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/24ba2fe2
Branch: refs/heads/fgdatamap
Commit: 24ba2fe2226f9168dcde6c216948f8656488293d
Parents: 8a86d3f
Author: chenliang613 <ch...@huawei.com>
Authored: Tue Jan 30 22:35:02 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 19:18:26 2018 +0800
----------------------------------------------------------------------
README.md | 12 +++----
docs/data-management-on-carbondata.md | 38 ++++++++++++++++++--
.../examples/StandardPartitionExample.scala | 7 ++--
.../preaggregate/TestPreAggCreateCommand.scala | 17 +++++++++
4 files changed, 61 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/24ba2fe2/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 15dba93..3b6792e 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@
<img src="/docs/images/CarbonData_logo.png" width="200" height="40">
-Apache CarbonData is an indexed columnar data format for fast analytics on big data platform, e.g.Apache Hadoop, Apache Spark, etc.
+Apache CarbonData is an indexed columnar data store solution for fast analytics on big data platform, e.g.Apache Hadoop, Apache Spark, etc.
You can find the latest CarbonData document and learn more at:
[http://carbondata.apache.org](http://carbondata.apache.org/)
@@ -25,14 +25,9 @@ You can find the latest CarbonData document and learn more at:
[CarbonData cwiki](https://cwiki.apache.org/confluence/display/CARBONDATA/)
## Status
-Spark2.1:
-[](https://builds.apache.org/view/A-D/view/CarbonData/job/carbondata-master-spark-2.1/badge/icon)
+Spark2.2:
+[](https://builds.apache.org/view/A-D/view/CarbonData/job/carbondata-master-spark-2.2/lastBuild/testReport)
[](https://coveralls.io/github/apache/carbondata?branch=master)
-## Features
-CarbonData file format is a columnar store in HDFS, it has many features that a modern columnar format has, such as splittable, compression schema ,complex data type etc, and CarbonData has following unique features:
-* Stores data along with index: it can significantly accelerate query performance and reduces the I/O scans and CPU resources, where there are filters in the query. CarbonData index consists of multiple level of indices, a processing framework can leverage this index to reduce the task it needs to schedule and process, and it can also do skip scan in more finer grain unit (called blocklet) in task side scanning instead of scanning the whole file.
-* Operable encoded data :Through supporting efficient compression and global encoding schemes, can query on compressed/encoded data, the data can be converted just before returning the results to the users, which is "late materialized".
-* Supports for various use cases with one single Data format : like interactive OLAP-style query, Sequential Access (big scan), Random Access (narrow scan).
## Building CarbonData
CarbonData is built using Apache Maven, to [build CarbonData](https://github.com/apache/carbondata/blob/master/build)
@@ -50,6 +45,7 @@ CarbonData is built using Apache Maven, to [build CarbonData](https://github.com
## Other Technical Material
[Apache CarbonData meetup material](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609)
+[Use Case Articles](https://cwiki.apache.org/confluence/display/CARBONDATA/CarbonData+Articles)
## Fork and Contribute
This is an active open source project for everyone, and we are always open to people who want to use this system or contribute to it.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/24ba2fe2/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index 3af95ac..d7954e1 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -567,9 +567,43 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
ALTER TABLE table_name COMPACT 'MAJOR'
```
-## PARTITION
+ - **CLEAN SEGMENTS AFTER Compaction**
+
+ Clean the segments which are compacted:
+ ```
+ CLEAN FILES FOR TABLE carbon_table
+ ```
+
+## STANDARD PARTITION
+
+ The partition is same as Spark, the creation partition command as below:
+
+ ```
+ CREATE TABLE [IF NOT EXISTS] [db_name.]table_name
+ [(col_name data_type , ...)]
+ PARTITIONED BY (partition_col_name data_type)
+ STORED BY 'carbondata'
+ [TBLPROPERTIES (property_name=property_value, ...)]
+ ```
+
+ Example:
+ ```
+ CREATE TABLE partitiontable0
+ (id Int,
+ vin String,
+ phonenumber Long,
+ area String,
+ salary Int)
+ PARTITIONED BY (country String)
+ STORED BY 'org.apache.carbondata.format'
+ TBLPROPERTIES('SORT_COLUMNS'='id,vin')
+ )
+ ```
+
+
+## CARBONDATA PARTITION(HASH,RANGE,LIST)
- Similar to other system's partition features, CarbonData's partition feature also can be used to improve query performance by filtering on the partition column.
+ The partition supports three type:(Hash,Range,List), similar to other system's partition features, CarbonData's partition feature can be used to improve query performance by filtering on the partition column.
### Create Hash Partition Table
http://git-wip-us.apache.org/repos/asf/carbondata/blob/24ba2fe2/examples/spark2/src/main/scala/org/apache/carbondata/examples/StandardPartitionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StandardPartitionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StandardPartitionExample.scala
index 5a8e3f5..1126ecc 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StandardPartitionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StandardPartitionExample.scala
@@ -47,6 +47,7 @@ object StandardPartitionExample {
| salary Int)
| PARTITIONED BY (country String)
| STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='id,vin')
""".stripMargin)
spark.sql(s"""
@@ -55,7 +56,7 @@ object StandardPartitionExample {
spark.sql(
s"""
- | SELECT *
+ | SELECT country,id,vin,phonenumver,area,salary
| FROM partitiontable0
""".stripMargin).show()
@@ -65,8 +66,8 @@ object StandardPartitionExample {
import scala.util.Random
import spark.implicits._
val r = new Random()
- val df = spark.sparkContext.parallelize(1 to 10 * 1000 * 1000)
- .map(x => ("No." + r.nextInt(100000), "country" + x % 8, "city" + x % 50, x % 300))
+ val df = spark.sparkContext.parallelize(1 to 10 * 1000 * 10)
+ .map(x => ("No." + r.nextInt(1000), "country" + x % 8, "city" + x % 50, x % 300))
.toDF("ID", "country", "city", "population")
// Create table without partition
http://git-wip-us.apache.org/repos/asf/carbondata/blob/24ba2fe2/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index 303abf4..23132de 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.carbondata.integration.spark.testsuite.preaggregate
import scala.collection.JavaConverters._
[39/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 07989b2..d5c2609 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -19,7 +19,13 @@ package org.apache.carbondata.core.metadata.schema.table;
import java.io.IOException;
import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -33,7 +39,10 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.model.QueryProjection;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeConverter;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -136,10 +145,7 @@ public class CarbonTable implements Serializable {
/**
* During creation of TableInfo from hivemetastore the DataMapSchemas and the columns
* DataTypes are not converted to the appropriate child classes.
- *
* This method will cast the same to the appropriate classes
- *
- * @param tableInfo
*/
private static void updateTableInfo(TableInfo tableInfo) {
List<DataMapSchema> dataMapSchemas = new ArrayList<>();
@@ -153,8 +159,9 @@ public class CarbonTable implements Serializable {
}
tableInfo.setDataMapSchemaList(dataMapSchemas);
for (ColumnSchema columnSchema : tableInfo.getFactTable().getListOfColumns()) {
- columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
- columnSchema.getPrecision(), columnSchema.getScale()));
+ columnSchema.setDataType(
+ DataTypeUtil.valueOf(
+ columnSchema.getDataType(), columnSchema.getPrecision(), columnSchema.getScale()));
}
List<DataMapSchema> childSchema = tableInfo.getDataMapSchemaList();
for (DataMapSchema dataMapSchema : childSchema) {
@@ -168,10 +175,11 @@ public class CarbonTable implements Serializable {
}
}
if (tableInfo.getFactTable().getBucketingInfo() != null) {
- for (ColumnSchema columnSchema : tableInfo.getFactTable()
- .getBucketingInfo().getListOfColumns()) {
- columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
- columnSchema.getPrecision(), columnSchema.getScale()));
+ for (ColumnSchema columnSchema :
+ tableInfo.getFactTable().getBucketingInfo().getListOfColumns()) {
+ columnSchema.setDataType(
+ DataTypeUtil.valueOf(
+ columnSchema.getDataType(), columnSchema.getPrecision(), columnSchema.getScale()));
}
}
if (tableInfo.getFactTable().getPartitionInfo() != null) {
@@ -211,6 +219,7 @@ public class CarbonTable implements Serializable {
/**
* fill columns as per user provided order
+ *
* @param tableName
*/
private void fillCreateOrderColumn(String tableName) {
@@ -229,7 +238,6 @@ public class CarbonTable implements Serializable {
this.createOrderColumn.put(tableName, columns);
}
-
/**
* Fill allDimensions and allMeasures for carbon table
*
@@ -254,8 +262,8 @@ public class CarbonTable implements Serializable {
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(),
+ -1, -1, ++complexTypeOrdinal);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
allDimensions.add(complexDimension);
dimensionOrdinal =
@@ -268,9 +276,8 @@ public class CarbonTable implements Serializable {
this.numberOfSortColumns++;
}
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
- CarbonDimension dimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ CarbonDimension dimension = new CarbonDimension(
+ columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), -1, -1, -1);
if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
this.numberOfNoDictSortColumns++;
}
@@ -278,25 +285,25 @@ public class CarbonTable implements Serializable {
primitiveDimensions.add(dimension);
} else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
&& columnSchema.getColumnGroupId() == -1) {
- CarbonDimension dimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1);
+ CarbonDimension dimension = new CarbonDimension(
+ columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), keyOrdinal++,
+ -1, -1);
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
} else {
columnGroupOrdinal =
previousColumnGroupId == columnSchema.getColumnGroupId() ? ++columnGroupOrdinal : 0;
previousColumnGroupId = columnSchema.getColumnGroupId();
- CarbonDimension dimension = new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), keyOrdinal++,
+ CarbonDimension dimension = new CarbonDimension(
+ columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), keyOrdinal++,
columnGroupOrdinal, -1);
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
}
}
} else {
- allMeasures.add(new CarbonMeasure(columnSchema, measureOrdinal++,
- columnSchema.getSchemaOrdinal()));
+ allMeasures.add(
+ new CarbonMeasure(columnSchema, measureOrdinal++, columnSchema.getSchemaOrdinal()));
}
}
fillVisibleDimensions(tableSchema.getTableName());
@@ -347,8 +354,8 @@ public class CarbonTable implements Serializable {
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(),
+ -1, -1, -1);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
parentDimension.getListOfChildDimensions().add(complexDimension);
dimensionOrdinal =
@@ -356,8 +363,8 @@ public class CarbonTable implements Serializable {
listOfColumns, complexDimension, primitiveDimensions);
} else {
CarbonDimension carbonDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(),
+ -1, -1, -1);
parentDimension.getListOfChildDimensions().add(carbonDimension);
primitiveDimensions.add(carbonDimension);
}
@@ -569,6 +576,7 @@ public class CarbonTable implements Serializable {
}
return null;
}
+
/**
* gets all children dimension for complex type
*
@@ -771,18 +779,13 @@ public class CarbonTable implements Serializable {
return dimensionOrdinalMax;
}
- public void setDimensionOrdinalMax(int dimensionOrdinalMax) {
- this.dimensionOrdinalMax = dimensionOrdinalMax;
- }
-
-
public boolean hasDataMapSchema() {
return hasDataMapSchema;
}
public boolean isChildDataMap() {
- return null != tableInfo.getParentRelationIdentifiers()
- && !tableInfo.getParentRelationIdentifiers().isEmpty();
+ return null != tableInfo.getParentRelationIdentifiers() &&
+ !tableInfo.getParentRelationIdentifiers().isEmpty();
}
/**
@@ -806,4 +809,63 @@ public class CarbonTable implements Serializable {
}
return dataSize + indexSize;
}
+
+ /**
+ * Create a new QueryModel with projection all columns in the table.
+ */
+ public QueryModel createQueryModelWithProjectAllColumns(DataTypeConverter converter) {
+ QueryProjection projection = new QueryProjection();
+
+ List<CarbonDimension> dimensions = getDimensionByTableName(getTableName());
+ for (int i = 0; i < dimensions.size(); i++) {
+ projection.addDimension(dimensions.get(i), i);
+ }
+ List<CarbonMeasure> measures = getMeasureByTableName(getTableName());
+ for (int i = 0; i < measures.size(); i++) {
+ projection.addMeasure(measures.get(i), i);
+ }
+ QueryModel model = QueryModel.newInstance(this);
+ model.setProjection(projection);
+ model.setConverter(converter);
+ return model;
+ }
+
+ /**
+ * Create a new QueryModel with specified projection
+ */
+ public QueryModel createQueryWithProjection(String[] projectionColumnNames,
+ DataTypeConverter converter) {
+ QueryProjection projection = createProjection(projectionColumnNames);
+ QueryModel queryModel = QueryModel.newInstance(this);
+ queryModel.setProjection(projection);
+ queryModel.setConverter(converter);
+ return queryModel;
+ }
+
+ private QueryProjection createProjection(String[] projectionColumnNames) {
+ String factTableName = getTableName();
+ QueryProjection projection = new QueryProjection();
+ // fill dimensions
+ // If columns are null, set all dimensions and measures
+ int i = 0;
+ if (projectionColumnNames != null) {
+ for (String projectionColumnName : projectionColumnNames) {
+ CarbonDimension dimension = getDimensionByName(factTableName, projectionColumnName);
+ if (dimension != null) {
+ projection.addDimension(dimension, i);
+ i++;
+ } else {
+ CarbonMeasure measure = getMeasureByName(factTableName, projectionColumnName);
+ if (measure == null) {
+ throw new RuntimeException(projectionColumnName +
+ " column not found in the table " + factTableName);
+ }
+ projection.addMeasure(measure, i);
+ i++;
+ }
+ }
+ }
+
+ return projection;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
index 20e539b..9a1dad1 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
@@ -26,18 +26,6 @@ import java.io.Serializable;
*/
public class RelationIdentifier implements Serializable, Writable {
- public void setDatabaseName(String databaseName) {
- this.databaseName = databaseName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public void setTableId(String tableId) {
- this.tableId = tableId;
- }
-
private String databaseName;
private String tableName;
@@ -50,10 +38,6 @@ public class RelationIdentifier implements Serializable, Writable {
this.tableId = tableId;
}
- public RelationIdentifier() {
- this(null, null, null);
- }
-
public String getDatabaseName() {
return databaseName;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
index 4deafd4..0d796c7 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
@@ -24,7 +24,11 @@ import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
index 13e7d49..c888418 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
@@ -45,7 +45,7 @@ public class CarbonColumn implements Serializable {
/**
* order in which user has created table
*/
- protected int schemaOrdinal;
+ private int schemaOrdinal;
/**
* Column identifier
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
index b4c052f..d15cde7 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
@@ -122,18 +122,6 @@ public class CarbonDimension extends CarbonColumn {
return getEncoder().contains(Encoding.DICTIONARY);
}
- public int getNumDimensionsExpanded() {
- if (listOfChildDimensions == null) {
- // there is no child, return 1 column
- return 1;
- }
- int columnCount = 1;
- for (CarbonDimension dimension: listOfChildDimensions) {
- columnCount += dimension.getNumDimensionsExpanded();
- }
- return columnCount;
- }
-
/**
* @return is column participated in sorting or not
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 4b99307..920155b 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -125,8 +125,6 @@ public class CarbonUpdateUtil {
lockStatus = updateLock.lockWithRetries();
if (lockStatus) {
- AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
-
// read the existing file if present and update the same.
SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager
.getUpdateStatusDetails();
@@ -367,16 +365,6 @@ public class CarbonUpdateUtil {
.substring(0, completeBlockName.lastIndexOf(CarbonCommonConstants.HYPHEN));
}
- /**
- * returns segment id from segment name
- *
- * @param segmentName
- * @return
- */
- public static String getSegmentId(String segmentName) {
- return segmentName.split(CarbonCommonConstants.UNDERSCORE)[1];
- }
-
public static long getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
String segmentDirPath = tablePath.getCarbonDataDirectoryPath(segmentId);
@@ -501,7 +489,7 @@ public class CarbonUpdateUtil {
// aborted scenario.
invalidDeleteDeltaFiles = updateStatusManager
- .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, false,
+ .getDeleteDeltaInvalidFilesList(block, false,
allSegmentFiles, isAbortedFile);
for (CarbonFile invalidFile : invalidDeleteDeltaFiles) {
boolean doForceDelete = true;
@@ -511,7 +499,7 @@ public class CarbonUpdateUtil {
// case 1
if (CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) {
completeListOfDeleteDeltaFiles = updateStatusManager
- .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, true,
+ .getDeleteDeltaInvalidFilesList(block, true,
allSegmentFiles, isInvalidFile);
for (CarbonFile invalidFile : completeListOfDeleteDeltaFiles) {
@@ -519,7 +507,7 @@ public class CarbonUpdateUtil {
}
CarbonFile[] blockRelatedFiles = updateStatusManager
- .getAllBlockRelatedFiles(block.getBlockName(), allSegmentFiles,
+ .getAllBlockRelatedFiles(allSegmentFiles,
block.getActualBlockName());
// now for each invalid index file need to check the query execution time out
@@ -533,7 +521,7 @@ public class CarbonUpdateUtil {
} else {
invalidDeleteDeltaFiles = updateStatusManager
- .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, false,
+ .getDeleteDeltaInvalidFilesList(block, false,
allSegmentFiles, isInvalidFile);
for (CarbonFile invalidFile : invalidDeleteDeltaFiles) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
index 4275cca..e5fdf6f 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
@@ -45,14 +45,6 @@ public class DeleteDeltaBlockDetails implements Serializable {
blockletDetails = new ArrayList<DeleteDeltaBlockletDetails>();
}
- public String getBlockName() {
- return blockName;
- }
-
- public void setBlockName(String blockName) {
- this.blockName = blockName;
- }
-
@Override public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null || !(obj instanceof DeleteDeltaBlockDetails)) return false;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
index 22785fa..6185134 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
@@ -46,14 +46,6 @@ public class DeleteDeltaBlockletDetails implements Serializable {
return deletedRows.add(row);
}
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
public Integer getPageId() {
return pageId;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
deleted file mode 100644
index 3c0dd4f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.mutate.data;
-
-import java.util.Map;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
-
-/**
- * This class is responsible for loading delete delta file cache based on
- * blocklet id of a particular block
- */
-public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderIntf {
- private String blockletID;
- private DataRefNode blockletNode;
- private AbsoluteTableIdentifier absoluteIdentifier;
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(BlockletDeleteDeltaCacheLoader.class.getName());
-
- public BlockletDeleteDeltaCacheLoader(String blockletID, DataRefNode blockletNode,
- AbsoluteTableIdentifier absoluteIdentifier) {
- this.blockletID = blockletID;
- this.blockletNode = blockletNode;
- this.absoluteIdentifier = absoluteIdentifier;
- }
-
- /**
- * This method will load the delete delta cache based on blocklet id of particular block with
- * the help of SegmentUpdateStatusManager.
- */
- public void loadDeleteDeltaFileDataToCache() {
- SegmentUpdateStatusManager segmentUpdateStatusManager =
- new SegmentUpdateStatusManager(absoluteIdentifier);
- Map<Integer, Integer[]> deleteDeltaFileData = null;
- BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache = null;
- if (null == blockletNode.getDeleteDeltaDataCache()) {
- try {
- deleteDeltaFileData =
- segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID);
- deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
- segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, null));
- } catch (Exception e) {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Unable to retrieve delete delta files");
- }
- }
- } else {
- deleteDeltaDataCache = blockletNode.getDeleteDeltaDataCache();
- // if already cache is present then validate the cache using timestamp
- String cacheTimeStamp = segmentUpdateStatusManager
- .getTimestampForRefreshCache(blockletID, deleteDeltaDataCache.getCacheTimeStamp());
- if (null != cacheTimeStamp) {
- try {
- deleteDeltaFileData =
- segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID);
- deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
- segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, cacheTimeStamp));
- } catch (Exception e) {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Unable to retrieve delete delta files");
- }
- }
- }
- }
- blockletNode.setDeleteDeltaDataCache(deleteDeltaDataCache);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
index b14660c..88b3065 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
@@ -18,7 +18,7 @@ package org.apache.carbondata.core.scan.collector;
import java.util.List;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
/**
@@ -27,17 +27,13 @@ import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
public interface ScannedResultCollector {
/**
- * Below method will be used to aggregate the scanned result
- *
- * @param scannedResult scanned result
- * @return how many records was aggregated
+ * Return the aggregated scanned result in list of rows
*/
- List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize);
+ List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize);
/**
- * Collects data in columnar format.
- * @param scannedResult
- * @param columnarBatch
+ * Collects result and set it in the specified columnar batch
*/
- void collectVectorBatch(AbstractScannedResult scannedResult, CarbonColumnarBatch columnarBatch);
+ void collectResultInColumnarBatch(BlockletScannedResult scannedResult,
+ CarbonColumnarBatch columnarBatch);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
index dc78ac6..694271e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
@@ -19,8 +19,6 @@ package org.apache.carbondata.core.scan.collector.impl;
import java.math.BigDecimal;
import java.math.RoundingMode;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
@@ -29,8 +27,8 @@ import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.DimensionInfo;
import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -39,38 +37,35 @@ import org.apache.carbondata.core.util.DataTypeUtil;
*/
public abstract class AbstractScannedResultCollector implements ScannedResultCollector {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractScannedResultCollector.class.getName());
-
/**
* table block execution infos
*/
- protected BlockExecutionInfo tableBlockExecutionInfos;
+ BlockExecutionInfo executionInfo;
/**
* maintains the measure information like datatype, ordinal, measure existence
*/
- protected MeasureInfo measureInfo;
+ MeasureInfo measureInfo;
/**
* maintains the dimension information like datatype, ordinal, measure existence
*/
- protected DimensionInfo dimensionInfo;
+ DimensionInfo dimensionInfo;
- public AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) {
- this.tableBlockExecutionInfos = blockExecutionInfos;
+ AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ this.executionInfo = blockExecutionInfos;
measureInfo = blockExecutionInfos.getMeasureInfo();
dimensionInfo = blockExecutionInfos.getDimensionInfo();
}
protected void fillMeasureData(Object[] msrValues, int offset,
- AbstractScannedResult scannedResult) {
+ BlockletScannedResult scannedResult) {
int measureExistIndex = 0;
for (short i = 0; i < measureInfo.getMeasureDataTypes().length; i++) {
// if measure exists is block then pass measure column
// data chunk to the collector
if (measureInfo.getMeasureExists()[i]) {
- QueryMeasure queryMeasure = tableBlockExecutionInfos.getQueryMeasures()[measureExistIndex];
+ ProjectionMeasure queryMeasure = executionInfo.getProjectionMeasures()[measureExistIndex];
msrValues[i + offset] = getMeasureData(
scannedResult.getMeasureChunk(measureInfo.getMeasureOrdinals()[measureExistIndex]),
scannedResult.getCurrentRowId(), queryMeasure.getMeasure());
@@ -87,8 +82,7 @@ public abstract class AbstractScannedResultCollector implements ScannedResultCol
}
}
- protected Object getMeasureData(ColumnPage dataChunk, int index,
- CarbonMeasure carbonMeasure) {
+ Object getMeasureData(ColumnPage dataChunk, int index, CarbonMeasure carbonMeasure) {
if (!dataChunk.getNullBits().get(index)) {
DataType dataType = carbonMeasure.getDataType();
if (dataType == DataTypes.BOOLEAN) {
@@ -114,7 +108,8 @@ public abstract class AbstractScannedResultCollector implements ScannedResultCol
return null;
}
- @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
+ @Override
+ public void collectResultInColumnarBatch(BlockletScannedResult scannedResult,
CarbonColumnarBatch columnarBatch) {
throw new UnsupportedOperationException("Works only for batch collectors");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index cf6e6d6..4322034 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -29,9 +29,9 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -42,49 +42,50 @@ import org.apache.commons.lang3.ArrayUtils;
*/
public class DictionaryBasedResultCollector extends AbstractScannedResultCollector {
- protected QueryDimension[] queryDimensions;
+ protected ProjectionDimension[] queryDimensions;
- protected QueryMeasure[] queryMeasures;
+ protected ProjectionMeasure[] queryMeasures;
- protected DirectDictionaryGenerator[] directDictionaryGenerators;
+ private DirectDictionaryGenerator[] directDictionaryGenerators;
/**
* query order
*/
protected int[] order;
- protected int[] actualIndexInSurrogateKey;
+ private int[] actualIndexInSurrogateKey;
- protected boolean[] dictionaryEncodingArray;
+ boolean[] dictionaryEncodingArray;
- protected boolean[] directDictionaryEncodingArray;
+ boolean[] directDictionaryEncodingArray;
- protected boolean[] implictColumnArray;
+ private boolean[] implictColumnArray;
- protected boolean[] complexDataTypeArray;
+ private boolean[] complexDataTypeArray;
- protected int dictionaryColumnIndex;
- protected int noDictionaryColumnIndex;
- protected int complexTypeColumnIndex;
+ int dictionaryColumnIndex;
+ int noDictionaryColumnIndex;
+ int complexTypeColumnIndex;
- protected boolean isDimensionExists;
+ boolean isDimensionExists;
- protected Map<Integer, GenericQueryType> comlexDimensionInfoMap;
+ private Map<Integer, GenericQueryType> comlexDimensionInfoMap;
public DictionaryBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
- queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
- queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ queryDimensions = executionInfo.getProjectionDimensions();
+ queryMeasures = executionInfo.getProjectionMeasures();
initDimensionAndMeasureIndexesForFillingData();
isDimensionExists = queryDimensions.length > 0;
- this.comlexDimensionInfoMap = tableBlockExecutionInfos.getComlexDimensionInfoMap();
+ this.comlexDimensionInfoMap = executionInfo.getComlexDimensionInfoMap();
}
/**
* This method will add a record both key and value to list object
* it will keep track of how many record is processed, to handle limit scenario
*/
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
// scan the record and add to list
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
@@ -118,13 +119,13 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
return listBasedResult;
}
- protected void fillDimensionData(AbstractScannedResult scannedResult, int[] surrogateResult,
+ void fillDimensionData(BlockletScannedResult scannedResult, int[] surrogateResult,
byte[][] noDictionaryKeys, byte[][] complexTypeKeyArray,
Map<Integer, GenericQueryType> comlexDimensionInfoMap, Object[] row, int i) {
if (!dictionaryEncodingArray[i]) {
if (implictColumnArray[i]) {
if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
- .equals(queryDimensions[i].getDimension().getColName())) {
+ .equals(queryDimensions[i].getColumnName())) {
row[order[i]] = DataTypeUtil.getDataBasedOnDataType(
scannedResult.getBlockletId() + CarbonCommonConstants.FILE_SEPARATOR + scannedResult
.getCurrentPageCounter() + CarbonCommonConstants.FILE_SEPARATOR + scannedResult
@@ -153,7 +154,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
}
}
- protected void fillMeasureData(AbstractScannedResult scannedResult, Object[] row) {
+ void fillMeasureData(BlockletScannedResult scannedResult, Object[] row) {
if (measureInfo.getMeasureDataTypes().length > 0) {
Object[] msrValues = new Object[measureInfo.getMeasureDataTypes().length];
fillMeasureData(msrValues, 0, scannedResult);
@@ -163,7 +164,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
}
}
- protected void initDimensionAndMeasureIndexesForFillingData() {
+ void initDimensionAndMeasureIndexesForFillingData() {
List<Integer> dictionaryIndexes = new ArrayList<Integer>();
for (int i = 0; i < queryDimensions.length; i++) {
if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY) || queryDimensions[i]
@@ -190,10 +191,10 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
complexDataTypeArray = CarbonUtil.getComplexDataTypeArray(queryDimensions);
order = new int[queryDimensions.length + queryMeasures.length];
for (int i = 0; i < queryDimensions.length; i++) {
- order[i] = queryDimensions[i].getQueryOrder();
+ order[i] = queryDimensions[i].getOrdinal();
}
for (int i = 0; i < queryMeasures.length; i++) {
- order[i + queryDimensions.length] = queryMeasures[i].getQueryOrder();
+ order[i + queryDimensions.length] = queryMeasures[i].getOrdinal();
}
directDictionaryGenerators = new DirectDictionaryGenerator[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
index 5e6c99a..e6cc817 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
@@ -23,9 +23,9 @@ import java.util.List;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
import org.apache.carbondata.core.scan.result.vector.MeasureDataVectorProcessor;
@@ -35,35 +35,35 @@ import org.apache.carbondata.core.scan.result.vector.MeasureDataVectorProcessor;
*/
public class DictionaryBasedVectorResultCollector extends AbstractScannedResultCollector {
- protected QueryDimension[] queryDimensions;
+ protected ProjectionDimension[] queryDimensions;
- protected QueryMeasure[] queryMeasures;
+ protected ProjectionMeasure[] queryMeasures;
- protected ColumnVectorInfo[] dictionaryInfo;
+ private ColumnVectorInfo[] dictionaryInfo;
- protected ColumnVectorInfo[] noDictionaryInfo;
+ private ColumnVectorInfo[] noDictionaryInfo;
- protected ColumnVectorInfo[] complexInfo;
+ private ColumnVectorInfo[] complexInfo;
- protected ColumnVectorInfo[] measureColumnInfo;
+ private ColumnVectorInfo[] measureColumnInfo;
- protected ColumnVectorInfo[] allColumnInfo;
+ ColumnVectorInfo[] allColumnInfo;
- protected ColumnVectorInfo[] implictColumnInfo;
+ private ColumnVectorInfo[] implictColumnInfo;
public DictionaryBasedVectorResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
// initialize only if the current block is not a restructured block else the initialization
// will be taken care by RestructureBasedVectorResultCollector
if (!blockExecutionInfos.isRestructuredBlock()) {
- queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
- queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ queryDimensions = executionInfo.getProjectionDimensions();
+ queryMeasures = executionInfo.getProjectionMeasures();
allColumnInfo = new ColumnVectorInfo[queryDimensions.length + queryMeasures.length];
prepareDimensionAndMeasureColumnVectors();
}
}
- protected void prepareDimensionAndMeasureColumnVectors() {
+ void prepareDimensionAndMeasureColumnVectors() {
measureColumnInfo = new ColumnVectorInfo[queryMeasures.length];
List<ColumnVectorInfo> dictInfoList = new ArrayList<>();
List<ColumnVectorInfo> noDictInfoList = new ArrayList<>();
@@ -78,13 +78,13 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
implictColumnList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
} else if (!queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
noDictInfoList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
} else if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
dictInfoList.add(columnVectorInfo);
@@ -92,21 +92,21 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
} else if (queryDimensions[i].getDimension().isComplex()) {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
complexList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
columnVectorInfo.genericQueryType =
- tableBlockExecutionInfos.getComlexDimensionInfoMap().get(columnVectorInfo.ordinal);
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ executionInfo.getComlexDimensionInfoMap().get(columnVectorInfo.ordinal);
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
} else {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
dictInfoList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
}
}
for (int i = 0; i < queryMeasures.length; i++) {
@@ -116,7 +116,7 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
columnVectorInfo.ordinal = queryMeasures[i].getMeasure().getOrdinal();
columnVectorInfo.measure = queryMeasures[i];
this.measureColumnInfo[i] = columnVectorInfo;
- allColumnInfo[queryMeasures[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryMeasures[i].getOrdinal()] = columnVectorInfo;
}
dictionaryInfo = dictInfoList.toArray(new ColumnVectorInfo[dictInfoList.size()]);
noDictionaryInfo = noDictInfoList.toArray(new ColumnVectorInfo[noDictInfoList.size()]);
@@ -126,11 +126,13 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
Arrays.sort(complexInfo);
}
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
- throw new UnsupportedOperationException("collectData is not supported here");
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
+ throw new UnsupportedOperationException("collectResultInRow is not supported here");
}
- @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
+ @Override
+ public void collectResultInColumnarBatch(BlockletScannedResult scannedResult,
CarbonColumnarBatch columnarBatch) {
int numberOfPages = scannedResult.numberOfpages();
int filteredRows = 0;
@@ -150,14 +152,15 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
return;
}
fillColumnVectorDetails(columnarBatch, rowCounter, requiredRows);
- filteredRows = scannedResult
- .markFilteredRows(columnarBatch, rowCounter, requiredRows, columnarBatch.getRowCounter());
- scanAndFillResult(scannedResult, columnarBatch, rowCounter, availableRows, requiredRows);
+ filteredRows = scannedResult.markFilteredRows(
+ columnarBatch, rowCounter, requiredRows, columnarBatch.getRowCounter());
+ fillResultToColumnarBatch(
+ scannedResult, columnarBatch, rowCounter, availableRows, requiredRows);
columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows - filteredRows);
}
}
- protected void scanAndFillResult(AbstractScannedResult scannedResult,
+ void fillResultToColumnarBatch(BlockletScannedResult scannedResult,
CarbonColumnarBatch columnarBatch, int rowCounter, int availableRows, int requiredRows) {
scannedResult.fillColumnarDictionaryBatch(dictionaryInfo);
scannedResult.fillColumnarNoDictionaryBatch(noDictionaryInfo);
@@ -174,8 +177,8 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
columnarBatch.setRowCounter(columnarBatch.getRowCounter() + requiredRows);
}
- protected void fillColumnVectorDetails(CarbonColumnarBatch columnarBatch, int rowCounter,
- int requiredRows) {
+ void fillColumnVectorDetails(CarbonColumnarBatch columnarBatch, int rowCounter, int requiredRows)
+ {
for (int i = 0; i < allColumnInfo.length; i++) {
allColumnInfo[i].size = requiredRows;
allColumnInfo[i].offset = rowCounter;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
index a09676a..0780675 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
@@ -16,14 +16,12 @@
*/
package org.apache.carbondata.core.scan.collector.impl;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
/**
@@ -31,15 +29,11 @@ import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
*/
public class RawBasedResultCollector extends AbstractScannedResultCollector {
- protected ByteArrayWrapper wrapper;
+ byte[] dictionaryKeyArray;
- protected byte[] dictionaryKeyArray;
+ byte[][] noDictionaryKeyArray;
- protected byte[][] noDictionaryKeyArray;
-
- protected byte[][] complexTypeKeyArray;
-
- protected byte[] implicitColumnByteArray;
+ private byte[][] complexTypeKeyArray;
public RawBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
@@ -49,9 +43,10 @@ public class RawBasedResultCollector extends AbstractScannedResultCollector {
* This method will add a record both key and value to list object
* it will keep track of how many record is processed, to handle limit scenario
*/
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
- QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ ProjectionMeasure[] queryMeasures = executionInfo.getProjectionMeasures();
// scan the record and add to list
int rowCounter = 0;
while (scannedResult.hasNext() && rowCounter < batchSize) {
@@ -65,24 +60,21 @@ public class RawBasedResultCollector extends AbstractScannedResultCollector {
return listBasedResult;
}
- protected void prepareRow(AbstractScannedResult scannedResult, List<Object[]> listBasedResult,
- QueryMeasure[] queryMeasures) {
+ void prepareRow(BlockletScannedResult scannedResult, List<Object[]> listBasedResult,
+ ProjectionMeasure[] queryMeasures) {
Object[] row = new Object[1 + queryMeasures.length];
- wrapper = new ByteArrayWrapper();
+ ByteArrayWrapper wrapper = new ByteArrayWrapper();
wrapper.setDictionaryKey(dictionaryKeyArray);
wrapper.setNoDictionaryKeys(noDictionaryKeyArray);
wrapper.setComplexTypesKeys(complexTypeKeyArray);
- wrapper.setImplicitColumnByteArray(implicitColumnByteArray);
row[0] = wrapper;
fillMeasureData(row, 1, scannedResult);
listBasedResult.add(row);
}
- protected void scanResultAndGetData(AbstractScannedResult scannedResult) {
+ void scanResultAndGetData(BlockletScannedResult scannedResult) {
dictionaryKeyArray = scannedResult.getDictionaryKeyArray();
noDictionaryKeyArray = scannedResult.getNoDictionaryKeyArray();
complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
- implicitColumnByteArray = scannedResult.getBlockletId()
- .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
index 296d9a6..8b42a4a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
@@ -23,8 +23,8 @@ import java.util.Map;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
/**
* class for handling restructure scenarios for filling result
@@ -35,8 +35,8 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
public RestructureBasedDictionaryResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
- queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
- queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures();
+ queryDimensions = executionInfo.getActualQueryDimensions();
+ queryMeasures = executionInfo.getActualQueryMeasures();
measureDefaultValues = new Object[queryMeasures.length];
fillMeasureDefaultValues();
initDimensionAndMeasureIndexesForFillingData();
@@ -61,7 +61,8 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
* This method will add a record both key and value to list object
* it will keep track of how many record is processed, to handle limit scenario
*/
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
// scan the record and add to list
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
int rowCounter = 0;
@@ -69,7 +70,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
byte[][] noDictionaryKeys;
byte[][] complexTypeKeyArray;
Map<Integer, GenericQueryType> comlexDimensionInfoMap =
- tableBlockExecutionInfos.getComlexDimensionInfoMap();
+ executionInfo.getComlexDimensionInfoMap();
while (scannedResult.hasNext() && rowCounter < batchSize) {
Object[] row = new Object[queryDimensions.length + queryMeasures.length];
if (isDimensionExists) {
@@ -107,13 +108,13 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
}
protected void fillMeasureData(Object[] msrValues, int offset,
- AbstractScannedResult scannedResult) {
+ BlockletScannedResult scannedResult) {
int measureExistIndex = 0;
for (short i = 0; i < measureInfo.getMeasureDataTypes().length; i++) {
// if measure exists is block then pass measure column
// data chunk to the collector
if (measureInfo.getMeasureExists()[i]) {
- QueryMeasure queryMeasure = tableBlockExecutionInfos.getQueryMeasures()[measureExistIndex];
+ ProjectionMeasure queryMeasure = executionInfo.getProjectionMeasures()[measureExistIndex];
msrValues[i + offset] = getMeasureData(
scannedResult.getMeasureChunk(measureInfo.getMeasureOrdinals()[measureExistIndex]),
scannedResult.getCurrentRowId(), queryMeasure.getMeasure());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
index b08a1d4..6544a75 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
@@ -30,9 +30,9 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.commons.lang3.ArrayUtils;
@@ -70,15 +70,15 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
*/
private void initRestructuredKeyGenerator() {
SegmentProperties segmentProperties =
- tableBlockExecutionInfos.getDataBlock().getSegmentProperties();
- QueryDimension[] queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
+ executionInfo.getDataBlock().getSegmentProperties();
+ ProjectionDimension[] queryDimensions = executionInfo.getActualQueryDimensions();
List<Integer> updatedColumnCardinality = new ArrayList<>(queryDimensions.length);
List<Integer> updatedDimensionPartitioner = new ArrayList<>(queryDimensions.length);
- int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex();
+ int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex();
int dimCounterInCurrentBlock = 0;
for (int i = 0; i < queryDimensions.length; i++) {
if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
- if (tableBlockExecutionInfos.getDimensionInfo().getDimensionExists()[i]) {
+ if (executionInfo.getDimensionInfo().getDimensionExists()[i]) {
// get the dictionary key ordinal as column cardinality in segment properties
// will only be for dictionary encoded columns
CarbonDimension currentBlockDimension = segmentProperties.getDimensions()
@@ -124,8 +124,8 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
*/
private void initCurrentBlockKeyGenerator() {
SegmentProperties segmentProperties =
- tableBlockExecutionInfos.getDataBlock().getSegmentProperties();
- int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex();
+ executionInfo.getDataBlock().getSegmentProperties();
+ int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex();
int[] updatedColumnCardinality = new int[dictionaryColumnBlockIndex.length];
int[] updatedDimensionPartitioner = new int[dictionaryColumnBlockIndex.length];
for (int i = 0; i < dictionaryColumnBlockIndex.length; i++) {
@@ -149,9 +149,10 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
* This method will add a record both key and value to list object
* it will keep track of how many record is processed, to handle limit scenario
*/
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
- QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures();
+ ProjectionMeasure[] queryMeasures = executionInfo.getActualQueryMeasures();
// scan the record and add to list
int rowCounter = 0;
while (scannedResult.hasNext() && rowCounter < batchSize) {
@@ -179,7 +180,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
* @return
*/
private byte[] fillDictionaryKeyArrayWithLatestSchema(byte[] dictionaryKeyArray) {
- QueryDimension[] actualQueryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
+ ProjectionDimension[] actualQueryDimensions = executionInfo.getActualQueryDimensions();
int newKeyArrayLength = dimensionInfo.getNewDictionaryColumnCount();
long[] keyArray = null;
if (null != updatedCurrentBlockKeyGenerator) {
@@ -222,7 +223,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
* @return
*/
private byte[][] fillNoDictionaryKeyArrayWithLatestSchema(byte[][] noDictionaryKeyArray) {
- QueryDimension[] actualQueryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
+ ProjectionDimension[] actualQueryDimensions = executionInfo.getActualQueryDimensions();
byte[][] noDictionaryKeyArrayWithNewlyAddedColumns =
new byte[noDictionaryKeyArray.length + dimensionInfo.getNewNoDictionaryColumnCount()][];
int existingColumnValueIndex = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
index 1e29e98..c54994a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
@@ -43,8 +43,8 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
public RestructureBasedVectorResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
- queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
- queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures();
+ queryDimensions = executionInfo.getActualQueryDimensions();
+ queryMeasures = executionInfo.getActualQueryMeasures();
measureDefaultValues = new Object[queryMeasures.length];
allColumnInfo = new ColumnVectorInfo[queryDimensions.length + queryMeasures.length];
createVectorForNewlyAddedDimensions();
@@ -66,7 +66,7 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
}
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
}
}
}
@@ -79,7 +79,7 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
if (!measureInfo.getMeasureExists()[i]) {
// add a dummy column vector result collector object
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
- allColumnInfo[queryMeasures[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryMeasures[i].getOrdinal()] = columnVectorInfo;
columnVectorInfo.measure = queryMeasures[i];
measureDefaultValues[i] = getMeasureDefaultValue(queryMeasures[i].getMeasure());
}
@@ -98,11 +98,13 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
- throw new UnsupportedOperationException("collectData is not supported here");
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
+ throw new UnsupportedOperationException("collectResultInRow is not supported here");
}
- @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
+ @Override
+ public void collectResultInColumnarBatch(BlockletScannedResult scannedResult,
CarbonColumnarBatch columnarBatch) {
int numberOfPages = scannedResult.numberOfpages();
while (scannedResult.getCurrentPageCounter() < numberOfPages) {
@@ -125,7 +127,8 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
fillDataForNonExistingDimensions();
fillDataForNonExistingMeasures();
// fill existing dimensions and measures data
- scanAndFillResult(scannedResult, columnarBatch, rowCounter, availableRows, requiredRows);
+ fillResultToColumnarBatch(
+ scannedResult, columnarBatch, rowCounter, availableRows, requiredRows);
columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows - filteredRows);
}
}
@@ -134,11 +137,11 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
* This method will fill the default values of non existing dimensions in the current block
*/
private void fillDataForNonExistingDimensions() {
- for (int i = 0; i < tableBlockExecutionInfos.getActualQueryDimensions().length; i++) {
+ for (int i = 0; i < executionInfo.getActualQueryDimensions().length; i++) {
if (!dimensionInfo.getDimensionExists()[i]) {
- int queryOrder = tableBlockExecutionInfos.getActualQueryDimensions()[i].getQueryOrder();
+ int queryOrder = executionInfo.getActualQueryDimensions()[i].getOrdinal();
CarbonDimension dimension =
- tableBlockExecutionInfos.getActualQueryDimensions()[i].getDimension();
+ executionInfo.getActualQueryDimensions()[i].getDimension();
if (dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
// fill direct dictionary column data
fillDirectDictionaryData(allColumnInfo[queryOrder].vector, allColumnInfo[queryOrder],
@@ -216,10 +219,10 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
* This method will fill the default values of non existing measures in the current block
*/
private void fillDataForNonExistingMeasures() {
- for (int i = 0; i < tableBlockExecutionInfos.getActualQueryMeasures().length; i++) {
+ for (int i = 0; i < executionInfo.getActualQueryMeasures().length; i++) {
if (!measureInfo.getMeasureExists()[i]) {
- int queryOrder = tableBlockExecutionInfos.getActualQueryMeasures()[i].getQueryOrder();
- CarbonMeasure measure = tableBlockExecutionInfos.getActualQueryMeasures()[i].getMeasure();
+ int queryOrder = executionInfo.getActualQueryMeasures()[i].getOrdinal();
+ CarbonMeasure measure = executionInfo.getActualQueryMeasures()[i].getMeasure();
ColumnVectorInfo columnVectorInfo = allColumnInfo[queryOrder];
CarbonColumnVector vector = columnVectorInfo.vector;
Object defaultValue = measureDefaultValues[i];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
index 1fcccca..30dd1dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
@@ -23,7 +23,7 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.spark.sql.catalyst.util.GenericArrayData;
import org.apache.spark.sql.types.ArrayType;
@@ -86,7 +86,7 @@ public class ArrayQueryType extends ComplexQueryType implements GenericQueryType
return new ArrayType(null, true);
}
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ @Override public void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder)
throws IOException {
readBlockDataChunk(blockChunkHolder);
children.fillRequiredBlockData(blockChunkHolder);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
index ee43a10..98f0715 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.scan.complextypes;
import java.io.IOException;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
public class ComplexQueryType {
protected String name;
@@ -42,7 +42,7 @@ public class ComplexQueryType {
protected byte[] copyBlockDataChunk(DimensionRawColumnChunk[] rawColumnChunks,
int rowNumber, int pageNumber) {
byte[] data =
- rawColumnChunks[blockIndex].convertToDimColDataChunk(pageNumber).getChunkData(rowNumber);
+ rawColumnChunks[blockIndex].decodeColumnPage(pageNumber).getChunkData(rowNumber);
byte[] output = new byte[data.length];
System.arraycopy(data, 0, output, 0, output.length);
return output;
@@ -51,10 +51,10 @@ public class ComplexQueryType {
/*
* This method will read the block data chunk from the respective block
*/
- protected void readBlockDataChunk(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ protected void readBlockDataChunk(RawBlockletColumnChunks blockChunkHolder) throws IOException {
+ if (null == blockChunkHolder.getDimensionRawColumnChunks()[blockIndex]) {
+ blockChunkHolder.getDimensionRawColumnChunks()[blockIndex] = blockChunkHolder.getDataBlock()
+ .readDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
index 978a652..b8aa912 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.keygenerator.mdkey.Bits;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.spark.sql.types.BooleanType$;
@@ -113,7 +113,7 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
}
}
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ @Override public void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder)
throws IOException {
readBlockDataChunk(blockChunkHolder);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
index 23a9f81..1d4f141 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
import org.apache.spark.sql.types.DataType;
@@ -106,7 +106,7 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
return new StructType(fields);
}
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ @Override public void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder)
throws IOException {
readBlockDataChunk(blockChunkHolder);
[04/50] [abbrv] carbondata git commit: [CARBONDATA-1224] Added page
level reader instead of reading whole blocklet in V3
Posted by ra...@apache.org.
[CARBONDATA-1224] Added page level reader instead of reading whole blocklet in V3
Problem:In V3 format we read the whole blocklet at once to memory in order save IO time. But it turns out to be costlier in case of parallel reading of more carbondata files.For example if we need to compact 50 segments then compactor need to open the readers on all the 50 segments to do merge sort. But the memory consumption is too high if each reader reads whole blocklet to the memory and there is high chances of going out of memory.Solution:In this type of scenarios we can introduce new readers for V3 to read the data page by page instead of reading whole blocklet at once to reduce the memory footprint.
This closes #1089
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d509f17f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d509f17f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d509f17f
Branch: refs/heads/fgdatamap
Commit: d509f17fbbf31b4baef23821f700bcbbfc987001
Parents: 181c280
Author: ravipesala <ra...@gmail.com>
Authored: Mon Jan 22 23:17:26 2018 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Tue Jan 30 22:12:09 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 11 ++
.../carbondata/core/datastore/FileHolder.java | 14 +-
.../datastore/chunk/AbstractRawColumnChunk.java | 6 +-
.../chunk/impl/DimensionRawColumnChunk.java | 17 +-
.../chunk/impl/MeasureRawColumnChunk.java | 18 +-
.../chunk/reader/CarbonDataReaderFactory.java | 22 ++-
...mpressedDimensionChunkFileBasedReaderV1.java | 2 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 2 +-
...essedDimChunkFileBasedPageLevelReaderV3.java | 168 +++++++++++++++++++
...mpressedDimensionChunkFileBasedReaderV3.java | 36 ++--
...CompressedMeasureChunkFileBasedReaderV1.java | 2 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 2 +-
...CompressedMeasureChunkFileBasedReaderV3.java | 39 ++---
...essedMsrChunkFileBasedPageLevelReaderV3.java | 153 +++++++++++++++++
.../core/datastore/impl/DFSFileHolderImpl.java | 16 +-
.../core/datastore/impl/FileHolderImpl.java | 19 +--
.../impl/btree/BlockletBTreeLeafNode.java | 54 +++++-
.../BlockletDataRefNodeWrapper.java | 38 +++--
.../executor/impl/AbstractQueryExecutor.java | 1 +
.../scan/executor/infos/BlockExecutionInfo.java | 13 ++
.../carbondata/core/scan/model/QueryModel.java | 13 ++
.../processor/AbstractDataBlockIterator.java | 25 ++-
.../core/scan/result/AbstractScannedResult.java | 69 +++++++-
.../AbstractDetailQueryResultIterator.java | 1 +
.../scan/scanner/AbstractBlockletScanner.java | 38 +++--
.../core/scan/scanner/impl/FilterScanner.java | 3 +-
.../apache/carbondata/core/util/CarbonUtil.java | 17 ++
.../merger/CarbonCompactionExecutor.java | 19 +++
28 files changed, 682 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index cf95dd9..77e8db8 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1556,6 +1556,17 @@ public final class CarbonCommonConstants {
// the maximum characters that can be supported should be less than Short max value
public static final int MAX_CHARS_PER_COLUMN_DEFAULT = 32000;
+ /**
+ * Enabling page level reader for compaction reduces the memory usage while compacting more
+ * number of segments. It allows reading only page by page instead of reaing whole blocklet to
+ * memory.
+ */
+ @CarbonProperty
+ public static final String CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION =
+ "carbon.enable.page.level.reader.in.compaction";
+
+ public static final String CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION_DEFAULT = "true";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
index 712e116..b4130a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.core.datastore;
-import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -103,6 +102,17 @@ public interface FileHolder {
String getQueryId();
- DataInputStream getDataInputStream(String filePath, long offset) throws IOException;
+ /**
+ * Set the flag to read data page by page instead of whole blocklet.
+ *
+ * @param isReadPageByPage
+ */
+ void setReadPageByPage(boolean isReadPageByPage);
+
+ /**
+ * Whether read the data page by page from carbondata file instead of reading whole
+ * blocklet to memory. It is need in case of memory constraint operations.
+ */
+ boolean isReadPageByPage();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java
index d1362c2..05ac9ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java
@@ -39,13 +39,13 @@ public abstract class AbstractRawColumnChunk {
protected int columnIndex;
- private int offSet;
+ private long offSet;
protected int length;
private DataChunk3 dataChunkV3;
- public AbstractRawColumnChunk(int columnIndex, ByteBuffer rawData, int offSet, int length) {
+ public AbstractRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length) {
this.columnIndex = columnIndex;
this.rawData = rawData;
this.offSet = offSet;
@@ -102,7 +102,7 @@ public abstract class AbstractRawColumnChunk {
return columnIndex;
}
- public int getOffSet() {
+ public long getOffSet() {
return offSet;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
index cb112c1..705c13c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
@@ -39,7 +39,7 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
private FileHolder fileHolder;
- public DimensionRawColumnChunk(int columnIndex, ByteBuffer rawData, int offSet, int length,
+ public DimensionRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
DimensionColumnChunkReader columnChunkReader) {
super(columnIndex, rawData, offSet, length);
this.chunkReader = columnChunkReader;
@@ -86,6 +86,21 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
return dataChunks[index];
}
+ /**
+ * Convert raw data with specified page number processed to DimensionColumnDataChunk
+ *
+ * @param index
+ * @return
+ */
+ public DimensionColumnDataChunk convertToDimColDataChunkWithOutCache(int index) {
+ assert index < pagesCount;
+ try {
+ return chunkReader.convertToDimensionChunk(this, index);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Override public void freeMemory() {
if (null != dataChunks) {
for (int i = 0; i < dataChunks.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
index d41cf09..fa0777b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
@@ -39,7 +39,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
private FileHolder fileReader;
- public MeasureRawColumnChunk(int columnIndex, ByteBuffer rawData, int offSet, int length,
+ public MeasureRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
MeasureColumnChunkReader chunkReader) {
super(columnIndex, rawData, offSet, length);
this.chunkReader = chunkReader;
@@ -85,6 +85,22 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
return columnPages[columnIndex];
}
+ /**
+ * Convert raw data with specified page number processed to MeasureColumnDataChunk
+ *
+ * @param index
+ * @return
+ */
+ public ColumnPage convertToColumnPageWithOutCache(int index) {
+ assert index < pagesCount;
+
+ try {
+ return chunkReader.convertToColumnPage(this, index);
+ } catch (IOException | MemoryException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Override public void freeMemory() {
if (null != columnPages) {
for (int i = 0; i < columnPages.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
index 8fee760..07cd7b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
@@ -18,10 +18,12 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.v2.CompressedDimensionChunkFileBasedReaderV2;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimChunkFileBasedPageLevelReaderV3;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimensionChunkFileBasedReaderV3;
import org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1;
import org.apache.carbondata.core.datastore.chunk.reader.measure.v2.CompressedMeasureChunkFileBasedReaderV2;
import org.apache.carbondata.core.datastore.chunk.reader.measure.v3.CompressedMeasureChunkFileBasedReaderV3;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.v3.CompressedMsrChunkFileBasedPageLevelReaderV3;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -62,7 +64,8 @@ public class CarbonDataReaderFactory {
* @return dimension column data reader based on version number
*/
public DimensionColumnChunkReader getDimensionColumnChunkReader(ColumnarFormatVersion version,
- BlockletInfo blockletInfo, int[] eachColumnValueSize, String filePath) {
+ BlockletInfo blockletInfo, int[] eachColumnValueSize, String filePath,
+ boolean readPagebyPage) {
switch (version) {
case V1:
return new CompressedDimensionChunkFileBasedReaderV1(blockletInfo, eachColumnValueSize,
@@ -72,8 +75,13 @@ public class CarbonDataReaderFactory {
filePath);
case V3:
default:
- return new CompressedDimensionChunkFileBasedReaderV3(blockletInfo, eachColumnValueSize,
- filePath);
+ if (readPagebyPage) {
+ return new CompressedDimChunkFileBasedPageLevelReaderV3(blockletInfo, eachColumnValueSize,
+ filePath);
+ } else {
+ return new CompressedDimensionChunkFileBasedReaderV3(blockletInfo, eachColumnValueSize,
+ filePath);
+ }
}
}
@@ -86,7 +94,7 @@ public class CarbonDataReaderFactory {
* @return measure column data reader based on version number
*/
public MeasureColumnChunkReader getMeasureColumnChunkReader(ColumnarFormatVersion version,
- BlockletInfo blockletInfo, String filePath) {
+ BlockletInfo blockletInfo, String filePath, boolean readPagebyPage) {
switch (version) {
case V1:
return new CompressedMeasureChunkFileBasedReaderV1(blockletInfo, filePath);
@@ -94,7 +102,11 @@ public class CarbonDataReaderFactory {
return new CompressedMeasureChunkFileBasedReaderV2(blockletInfo, filePath);
case V3:
default:
- return new CompressedMeasureChunkFileBasedReaderV3(blockletInfo, filePath);
+ if (readPagebyPage) {
+ return new CompressedMsrChunkFileBasedPageLevelReaderV3(blockletInfo, filePath);
+ } else {
+ return new CompressedMeasureChunkFileBasedReaderV3(blockletInfo, filePath);
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index 3e45082..27a4d89 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -108,7 +108,7 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
FileHolder fileReader = dimensionRawColumnChunk.getFileReader();
ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
- dataPage = COMPRESSOR.unCompressByte(rawData.array(), dimensionRawColumnChunk.getOffSet(),
+ dataPage = COMPRESSOR.unCompressByte(rawData.array(), (int) dimensionRawColumnChunk.getOffSet(),
dimensionRawColumnChunk.getLength());
// if row id block is present then read the row id chunk and uncompress it
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index 0dea099..b43f89c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -122,7 +122,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
int[] invertedIndexesReverse = null;
int[] rlePage = null;
DataChunk2 dimensionColumnChunk = null;
- int copySourcePoint = dimensionRawColumnChunk.getOffSet();
+ int copySourcePoint = (int) dimensionRawColumnChunk.getOffSet();
int blockIndex = dimensionRawColumnChunk.getColumnIndex();
ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
if (dimensionChunksOffset.size() - 1 == blockIndex) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
new file mode 100644
index 0000000..1edfd09
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.reader.dimension.v3;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.format.DataChunk2;
+import org.apache.carbondata.format.DataChunk3;
+import org.apache.carbondata.format.Encoding;
+
+/**
+ * Dimension column V3 Reader class which will be used to read and uncompress
+ * V3 format data. It reads the data in each page at once unlike whole blocklet. It is
+ * used for memory constraint operations like compaction.
+ * data format
+ * Data Format
+ * <FileHeader>
+ * <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
+ * <Column2 Data ChunkV3><Column2<Page1><Page2><Page3><Page4>>
+ * <Column3 Data ChunkV3><Column3<Page1><Page2><Page3><Page4>>
+ * <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
+ * <File Footer>
+ */
+public class CompressedDimChunkFileBasedPageLevelReaderV3
+ extends CompressedDimensionChunkFileBasedReaderV3 {
+
+ /**
+ * end position of last dimension in carbon data file
+ */
+ private long lastDimensionOffsets;
+
+ public CompressedDimChunkFileBasedPageLevelReaderV3(BlockletInfo blockletInfo,
+ int[] eachColumnValueSize, String filePath) {
+ super(blockletInfo, eachColumnValueSize, filePath);
+ lastDimensionOffsets = blockletInfo.getDimensionOffset();
+ }
+
+ /**
+ * Below method will be used to read the dimension column data form carbon data file
+ * Steps for reading
+ * 1. Get the length of the data to be read
+ * 2. Allocate the direct buffer
+ * 3. read the data from file
+ * 4. Get the data chunk object from data read
+ * 5. Create the raw chunk object and fill the details
+ *
+ * @param fileReader reader for reading the column from carbon data file
+ * @param blockletColumnIndex blocklet index of the column in carbon data file
+ * @return dimension raw chunk
+ */
+ public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader,
+ int blockletColumnIndex) throws IOException {
+ // get the current dimension offset
+ long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex);
+ int length = 0;
+ // to calculate the length of the data to be read
+ // column other than last column we can subtract the offset of current column with
+ // next column and get the total length.
+ // but for last column we need to use lastDimensionOffset which is the end position
+ // of the last dimension, we can subtract current dimension offset from lastDimesionOffset
+ if (dimensionChunksOffset.size() - 1 == blockletColumnIndex) {
+ length = (int) (lastDimensionOffsets - currentDimensionOffset);
+ } else {
+ length = (int) (dimensionChunksOffset.get(blockletColumnIndex + 1) - currentDimensionOffset);
+ }
+ ByteBuffer buffer;
+ // read the data from carbon data file
+ synchronized (fileReader) {
+ buffer = fileReader.readByteBuffer(filePath, currentDimensionOffset,
+ dimensionChunksLength.get(blockletColumnIndex));
+ }
+ // get the data chunk which will have all the details about the data pages
+ DataChunk3 dataChunk = CarbonUtil.readDataChunk3(new ByteArrayInputStream(buffer.array()));
+ DimensionRawColumnChunk rawColumnChunk =
+ getDimensionRawColumnChunk(fileReader, blockletColumnIndex, currentDimensionOffset, length,
+ null, dataChunk);
+
+ return rawColumnChunk;
+ }
+
+ /**
+ * Below method will be used to read the multiple dimension column data in group
+ * and divide into dimension raw chunk object
+ * Steps for reading
+ * 1. Get the length of the data to be read
+ * 2. Allocate the direct buffer
+ * 3. read the data from file
+ * 4. Get the data chunk object from file for each column
+ * 5. Create the raw chunk object and fill the details for each column
+ * 6. increment the offset of the data
+ *
+ * @param fileReader reader which will be used to read the dimension columns data from file
+ * @param startBlockletColumnIndex blocklet index of the first dimension column
+ * @param endBlockletColumnIndex blocklet index of the last dimension column
+ * @ DimensionRawColumnChunk array
+ */
+ protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException {
+ // create raw chunk for each dimension column
+ DimensionRawColumnChunk[] dimensionDataChunks =
+ new DimensionRawColumnChunk[endBlockletColumnIndex - startBlockletColumnIndex + 1];
+ int index = 0;
+ for (int i = startBlockletColumnIndex; i <= endBlockletColumnIndex; i++) {
+ dimensionDataChunks[index] = readRawDimensionChunk(fileReader, i);
+ index++;
+ }
+ return dimensionDataChunks;
+ }
+
+ /**
+ * Below method will be used to convert the compressed dimension chunk raw data to actual data
+ *
+ * @param dimensionRawColumnChunk dimension raw chunk
+ * @param pageNumber number
+ * @return DimensionColumnDataChunk
+ */
+ @Override public DimensionColumnDataChunk convertToDimensionChunk(
+ DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber)
+ throws IOException, MemoryException {
+ // data chunk of page
+ DataChunk2 pageMetadata = null;
+ // data chunk of blocklet column
+ DataChunk3 dataChunk3 = dimensionRawColumnChunk.getDataChunkV3();
+
+ pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber);
+ // calculating the start point of data
+ // as buffer can contain multiple column data, start point will be datachunkoffset +
+ // data chunk length + page offset
+ long offset = dimensionRawColumnChunk.getOffSet() + dimensionChunksLength
+ .get(dimensionRawColumnChunk.getColumnIndex()) + dataChunk3.getPage_offset()
+ .get(pageNumber);
+ int length = pageMetadata.data_page_length;
+ if (hasEncoding(pageMetadata.encoders, Encoding.INVERTED_INDEX)) {
+ length += pageMetadata.rowid_page_length;
+ }
+
+ if (hasEncoding(pageMetadata.encoders, Encoding.RLE)) {
+ length += pageMetadata.rle_page_length;
+ }
+ // get the data buffer
+ ByteBuffer rawData = dimensionRawColumnChunk.getFileReader()
+ .readByteBuffer(filePath, offset, length);
+
+ return decodeDimension(dimensionRawColumnChunk, rawData, pageMetadata, 0);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
index bb828a6..566e9b7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
@@ -103,9 +103,15 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
}
// get the data chunk which will have all the details about the data pages
DataChunk3 dataChunk = CarbonUtil.readDataChunk3(buffer, 0, length);
+ return getDimensionRawColumnChunk(fileReader, blockletColumnIndex, 0, length, buffer,
+ dataChunk);
+ }
+
+ protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileHolder fileReader,
+ int blockletColumnIndex, long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) {
// creating a raw chunks instance and filling all the details
DimensionRawColumnChunk rawColumnChunk =
- new DimensionRawColumnChunk(blockletColumnIndex, buffer, 0, length, this);
+ new DimensionRawColumnChunk(blockletColumnIndex, buffer, offset, length, this);
int numberOfPages = dataChunk.getPage_length().size();
byte[][] maxValueOfEachPage = new byte[numberOfPages][];
byte[][] minValueOfEachPage = new byte[numberOfPages][];
@@ -166,29 +172,11 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
int runningLength = 0;
for (int i = startBlockletColumnIndex; i <= endBlockletColumnIndex; i++) {
int currentLength = (int) (dimensionChunksOffset.get(i + 1) - dimensionChunksOffset.get(i));
- dimensionDataChunks[index] =
- new DimensionRawColumnChunk(i, buffer, runningLength, currentLength, this);
DataChunk3 dataChunk =
CarbonUtil.readDataChunk3(buffer, runningLength, dimensionChunksLength.get(i));
- int numberOfPages = dataChunk.getPage_length().size();
- byte[][] maxValueOfEachPage = new byte[numberOfPages][];
- byte[][] minValueOfEachPage = new byte[numberOfPages][];
- int[] eachPageLength = new int[numberOfPages];
- for (int j = 0; j < minValueOfEachPage.length; j++) {
- maxValueOfEachPage[j] =
- dataChunk.getData_chunk_list().get(j).getMin_max().getMax_values().get(0).array();
- minValueOfEachPage[j] =
- dataChunk.getData_chunk_list().get(j).getMin_max().getMin_values().get(0).array();
- eachPageLength[j] = dataChunk.getData_chunk_list().get(j).getNumberOfRowsInpage();
- }
- dimensionDataChunks[index].setDataChunkV3(dataChunk);
- dimensionDataChunks[index].setFileHolder(fileReader);
- dimensionDataChunks[index].setPagesCount(dataChunk.getPage_length().size());
- dimensionDataChunks[index].setMaxValues(maxValueOfEachPage);
- dimensionDataChunks[index].setMinValues(minValueOfEachPage);
- dimensionDataChunks[index].setRowCount(eachPageLength);
- dimensionDataChunks[index].setOffsets(ArrayUtils
- .toPrimitive(dataChunk.page_offset.toArray(new Integer[dataChunk.page_offset.size()])));
+ dimensionDataChunks[index] =
+ getDimensionRawColumnChunk(fileReader, i, runningLength, currentLength, buffer,
+ dataChunk);
runningLength += currentLength;
index++;
}
@@ -212,7 +200,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
// calculating the start point of data
// as buffer can contain multiple column data, start point will be datachunkoffset +
// data chunk length + page offset
- int offset = rawColumnPage.getOffSet() + dimensionChunksLength
+ int offset = (int) rawColumnPage.getOffSet() + dimensionChunksLength
.get(rawColumnPage.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber);
// first read the data and uncompressed it
return decodeDimension(rawColumnPage, rawData, pageMetadata, offset);
@@ -240,7 +228,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
return false;
}
- private DimensionColumnDataChunk decodeDimension(DimensionRawColumnChunk rawColumnPage,
+ protected DimensionColumnDataChunk decodeDimension(DimensionRawColumnChunk rawColumnPage,
ByteBuffer pageData, DataChunk2 pageMetadata, int offset)
throws IOException, MemoryException {
if (isEncodedWithMeta(pageMetadata)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index fcfd862..ae55375 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -98,7 +98,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta);
ColumnPage decodedPage = codec.decode(measureRawColumnChunk.getRawData().array(),
- measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength());
+ (int) measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength());
decodedPage.setNullBits(dataChunk.getNullValueIndexForColumn());
return decodedPage;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 001c240..2ddc202 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -110,7 +110,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
int pageNumber) throws IOException, MemoryException {
- int copyPoint = measureRawColumnChunk.getOffSet();
+ int copyPoint = (int) measureRawColumnChunk.getOffSet();
int blockIndex = measureRawColumnChunk.getColumnIndex();
ByteBuffer rawData = measureRawColumnChunk.getRawData();
DataChunk2 measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
index e207c82..4f6987b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -94,9 +94,17 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
// get the data chunk which will have all the details about the data pages
DataChunk3 dataChunk =
CarbonUtil.readDataChunk3(buffer, 0, measureColumnChunkLength.get(columnIndex));
+
+ return getMeasureRawColumnChunk(fileReader, columnIndex, 0, dataLength, buffer,
+ dataChunk);
+ }
+
+ protected MeasureRawColumnChunk getMeasureRawColumnChunk(FileHolder fileReader,
+ int columnIndex, long offset, int dataLength, ByteBuffer buffer,
+ DataChunk3 dataChunk) {
// creating a raw chunks instance and filling all the details
MeasureRawColumnChunk rawColumnChunk =
- new MeasureRawColumnChunk(columnIndex, buffer, 0, dataLength, this);
+ new MeasureRawColumnChunk(columnIndex, buffer, offset, dataLength, this);
int numberOfPages = dataChunk.getPage_length().size();
byte[][] maxValueOfEachPage = new byte[numberOfPages][];
byte[][] minValueOfEachPage = new byte[numberOfPages][];
@@ -158,31 +166,10 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
int currentLength =
(int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i));
- MeasureRawColumnChunk measureRawColumnChunk =
- new MeasureRawColumnChunk(i, buffer, runningLength, currentLength, this);
DataChunk3 dataChunk =
CarbonUtil.readDataChunk3(buffer, runningLength, measureColumnChunkLength.get(i));
-
- int numberOfPages = dataChunk.getPage_length().size();
- byte[][] maxValueOfEachPage = new byte[numberOfPages][];
- byte[][] minValueOfEachPage = new byte[numberOfPages][];
- int[] eachPageLength = new int[numberOfPages];
- for (int j = 0; j < minValueOfEachPage.length; j++) {
- maxValueOfEachPage[j] =
- dataChunk.getData_chunk_list().get(j).getMin_max().getMax_values().get(0).array();
- minValueOfEachPage[j] =
- dataChunk.getData_chunk_list().get(j).getMin_max().getMin_values().get(0).array();
- eachPageLength[j] = dataChunk.getData_chunk_list().get(j).getNumberOfRowsInpage();
- }
- measureRawColumnChunk.setDataChunkV3(dataChunk);
- ;
- measureRawColumnChunk.setFileReader(fileReader);
- measureRawColumnChunk.setPagesCount(dataChunk.getPage_length().size());
- measureRawColumnChunk.setMaxValues(maxValueOfEachPage);
- measureRawColumnChunk.setMinValues(minValueOfEachPage);
- measureRawColumnChunk.setRowCount(eachPageLength);
- measureRawColumnChunk.setOffsets(ArrayUtils
- .toPrimitive(dataChunk.page_offset.toArray(new Integer[dataChunk.page_offset.size()])));
+ MeasureRawColumnChunk measureRawColumnChunk =
+ getMeasureRawColumnChunk(fileReader, i, runningLength, currentLength, buffer, dataChunk);
measureDataChunk[index] = measureRawColumnChunk;
runningLength += currentLength;
index++;
@@ -208,7 +195,7 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
// calculating the start point of data
// as buffer can contain multiple column data, start point will be datachunkoffset +
// data chunk length + page offset
- int offset = rawColumnPage.getOffSet() +
+ int offset = (int) rawColumnPage.getOffSet() +
measureColumnChunkLength.get(rawColumnPage.getColumnIndex()) +
dataChunk3.getPage_offset().get(pageNumber);
ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnPage.getRawData(), offset);
@@ -219,7 +206,7 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
/**
* Decode measure column page with page header and raw data starting from offset
*/
- private ColumnPage decodeMeasure(DataChunk2 pageMetadata, ByteBuffer pageData, int offset)
+ protected ColumnPage decodeMeasure(DataChunk2 pageMetadata, ByteBuffer pageData, int offset)
throws MemoryException, IOException {
List<Encoding> encodings = pageMetadata.getEncoders();
List<ByteBuffer> encoderMetas = pageMetadata.getEncoder_meta();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
new file mode 100644
index 0000000..31ff4c0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.reader.measure.v3;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.format.DataChunk2;
+import org.apache.carbondata.format.DataChunk3;
+
+/**
+ * Measure column V3 Reader class which will be used to read and uncompress
+ * V3 format data
+ * data format
+ * Data Format
+ * <FileHeader>
+ * <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
+ * <Column2 Data ChunkV3><Column2<Page1><Page2><Page3><Page4>>
+ * <Column3 Data ChunkV3><Column3<Page1><Page2><Page3><Page4>>
+ * <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
+ * <File Footer>
+ */
+public class CompressedMsrChunkFileBasedPageLevelReaderV3
+ extends CompressedMeasureChunkFileBasedReaderV3 {
+
+ /**
+ * end position of last measure in carbon data file
+ */
+ private long measureOffsets;
+
+ public CompressedMsrChunkFileBasedPageLevelReaderV3(BlockletInfo blockletInfo, String filePath) {
+ super(blockletInfo, filePath);
+ measureOffsets = blockletInfo.getMeasureOffsets();
+ }
+
+ /**
+ * Below method will be used to read the measure column data form carbon data file
+ * 1. Get the length of the data to be read
+ * 2. Allocate the direct buffer
+ * 3. read the data from file
+ * 4. Get the data chunk object from data read
+ * 5. Create the raw chunk object and fill the details
+ *
+ * @param fileReader reader for reading the column from carbon data file
+ * @param blockletColumnIndex blocklet index of the column in carbon data file
+ * @return measure raw chunk
+ */
+ @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader,
+ int blockletColumnIndex) throws IOException {
+ int dataLength = 0;
+ // to calculate the length of the data to be read
+ // column other than last column we can subtract the offset of current column with
+ // next column and get the total length.
+ // but for last column we need to use lastDimensionOffset which is the end position
+ // of the last dimension, we can subtract current dimension offset from lastDimesionOffset
+ if (measureColumnChunkOffsets.size() - 1 == blockletColumnIndex) {
+ dataLength = (int) (measureOffsets - measureColumnChunkOffsets.get(blockletColumnIndex));
+ } else {
+ dataLength =
+ (int) (measureColumnChunkOffsets.get(blockletColumnIndex + 1) - measureColumnChunkOffsets
+ .get(blockletColumnIndex));
+ }
+ ByteBuffer buffer;
+ // read the data from carbon data file
+ synchronized (fileReader) {
+ buffer = fileReader
+ .readByteBuffer(filePath, measureColumnChunkOffsets.get(blockletColumnIndex),
+ measureColumnChunkLength.get(blockletColumnIndex));
+ }
+ // get the data chunk which will have all the details about the data pages
+ DataChunk3 dataChunk = CarbonUtil.readDataChunk3(new ByteArrayInputStream(buffer.array()));
+ return getMeasureRawColumnChunk(fileReader, blockletColumnIndex,
+ measureColumnChunkOffsets.get(blockletColumnIndex), dataLength, null, dataChunk);
+ }
+
+ /**
+ * Below method will be used to read the multiple measure column data in group
+ * and divide into measure raw chunk object
+ * Steps for reading
+ * 1. Get the length of the data to be read
+ * 2. Allocate the direct buffer
+ * 3. read the data from file
+ * 4. Get the data chunk object from file for each column
+ * 5. Create the raw chunk object and fill the details for each column
+ * 6. increment the offset of the data
+ *
+ * @param fileReader reader which will be used to read the measure columns data from file
+ * @param startColumnBlockletIndex blocklet index of the first measure column
+ * @param endColumnBlockletIndex blocklet index of the last measure column
+ * @return MeasureRawColumnChunk array
+ */
+ protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
+ int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
+ // create raw chunk for each measure column
+ MeasureRawColumnChunk[] measureDataChunk =
+ new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1];
+ int index = 0;
+ for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
+ measureDataChunk[index] = readRawMeasureChunk(fileReader, i);
+ index++;
+ }
+ return measureDataChunk;
+ }
+
+ /**
+ * Below method will be used to convert the compressed measure chunk raw data to actual data
+ *
+ * @param rawColumnPage measure raw chunk
+ * @param pageNumber number
+ * @return DimensionColumnDataChunk
+ */
+ @Override public ColumnPage convertToColumnPage(
+ MeasureRawColumnChunk rawColumnPage, int pageNumber)
+ throws IOException, MemoryException {
+ // data chunk of blocklet column
+ DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3();
+ // data chunk of page
+ DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber);
+ // calculating the start point of data
+ // as buffer can contain multiple column data, start point will be datachunkoffset +
+ // data chunk length + page offset
+ long offset = rawColumnPage.getOffSet() + measureColumnChunkLength
+ .get(rawColumnPage.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber);
+ ByteBuffer buffer = rawColumnPage.getFileReader()
+ .readByteBuffer(filePath, offset, pageMetadata.data_page_length);
+
+ ColumnPage decodedPage = decodeMeasure(pageMetadata, buffer, 0);
+ decodedPage.setNullBits(getNullBitSet(pageMetadata.presence));
+ return decodedPage;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
index 3032ec2..182c8eb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
@@ -16,8 +16,6 @@
*/
package org.apache.carbondata.core.datastore.impl;
-import java.io.BufferedInputStream;
-import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
@@ -39,6 +37,8 @@ public class DFSFileHolderImpl implements FileHolder {
private String queryId;
+ private boolean readPageByPage;
+
public DFSFileHolderImpl() {
this.fileNameAndStreamCache =
@@ -151,12 +151,14 @@ public class DFSFileHolderImpl implements FileHolder {
return queryId;
}
- @Override public DataInputStream getDataInputStream(String filePath, long offset)
- throws IOException {
- FSDataInputStream fsDataInputStream = updateCache(filePath);
- fsDataInputStream.seek(offset);
- return new DataInputStream(new BufferedInputStream(fsDataInputStream, 1 * 1024 * 1024));
+ @Override public void setReadPageByPage(boolean isReadPageByPage) {
+ this.readPageByPage = isReadPageByPage;
}
+
+ @Override public boolean isReadPageByPage() {
+ return readPageByPage;
+ }
+
public Map<String, FSDataInputStream> getFileNameAndStreamCache() {
return fileNameAndStreamCache;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
index c867f75..cc589b7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.core.datastore.impl;
-import java.io.BufferedInputStream;
-import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -38,6 +36,8 @@ public class FileHolderImpl implements FileHolder {
private Map<String, FileChannel> fileNameAndStreamCache;
private String queryId;
+ private boolean readPageByPage;
+
/**
* FileHolderImpl Constructor
* It will create the cache
@@ -214,16 +214,11 @@ public class FileHolderImpl implements FileHolder {
return queryId;
}
- @Override public DataInputStream getDataInputStream(String filePath, long offset)
- throws IOException {
- FileInputStream stream = new FileInputStream(filePath);
- long skipped = stream.skip(offset);
- long toSkip = offset - skipped;
- while (toSkip <= 0) {
- skipped = stream.skip(toSkip);
- toSkip = toSkip - skipped;
- }
+ @Override public void setReadPageByPage(boolean isReadPageByPage) {
+ this.readPageByPage = isReadPageByPage;
+ }
- return new DataInputStream(new BufferedInputStream(stream));
+ @Override public boolean isReadPageByPage() {
+ return readPageByPage;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
index 82c4169..94221ba 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -44,6 +44,16 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
private MeasureColumnChunkReader measureColumnChunkReader;
/**
+ * reader for dimension chunk of page level
+ */
+ private DimensionColumnChunkReader dimensionChunksPageLevelReader;
+
+ /**
+ * reader of measure chunk of page level
+ */
+ private MeasureColumnChunkReader measureColumnChunkPageLevelReader;
+
+ /**
* number of pages in blocklet
*/
private int numberOfPages;
@@ -76,12 +86,28 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
.getDimensionColumnChunkReader(builderInfos.getFooterList().get(0).getVersionId(),
builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex),
builderInfos.getDimensionColumnValueSize(),
- builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath());
+ builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath(),
+ false);
// create a instance of measure column chunk reader
measureColumnChunkReader = CarbonDataReaderFactory.getInstance()
.getMeasureColumnChunkReader(builderInfos.getFooterList().get(0).getVersionId(),
builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex),
- builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath());
+ builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath(),
+ false);
+ // create a instance of dimension chunk
+ dimensionChunksPageLevelReader = CarbonDataReaderFactory.getInstance()
+ .getDimensionColumnChunkReader(builderInfos.getFooterList().get(0).getVersionId(),
+ builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex),
+ builderInfos.getDimensionColumnValueSize(),
+ builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath(),
+ true);
+ // create a instance of measure column chunk reader
+ measureColumnChunkPageLevelReader = CarbonDataReaderFactory.getInstance()
+ .getMeasureColumnChunkReader(builderInfos.getFooterList().get(0).getVersionId(),
+ builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex),
+ builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath(),
+ true);
+
this.nodeNumber = nodeNumber;
this.numberOfPages =
builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex).getNumberOfPages();
@@ -111,7 +137,11 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
*/
@Override public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader,
int[][] blockIndexes) throws IOException {
- return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
+ if (fileReader.isReadPageByPage()) {
+ return dimensionChunksPageLevelReader.readRawDimensionChunks(fileReader, blockIndexes);
+ } else {
+ return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
+ }
}
/**
@@ -123,7 +153,11 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
*/
@Override public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndex)
throws IOException {
- return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndex);
+ if (fileReader.isReadPageByPage()) {
+ return dimensionChunksPageLevelReader.readRawDimensionChunk(fileReader, blockIndex);
+ } else {
+ return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndex);
+ }
}
/**
@@ -135,7 +169,11 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
*/
@Override public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader,
int[][] blockIndexes) throws IOException {
- return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
+ if (fileReader.isReadPageByPage()) {
+ return measureColumnChunkPageLevelReader.readRawMeasureChunks(fileReader, blockIndexes);
+ } else {
+ return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
+ }
}
/**
@@ -147,7 +185,11 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
*/
@Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
throws IOException {
- return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
+ if (fileReader.isReadPageByPage()) {
+ return measureColumnChunkPageLevelReader.readRawMeasureChunk(fileReader, blockIndex);
+ } else {
+ return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
index 1585414..097dd8c 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -117,46 +117,56 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
@Override
public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader, int[][] blockIndexes)
throws IOException {
- DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader();
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
}
@Override
public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndexes)
throws IOException {
- DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader();
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndexes);
}
@Override
public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
throws IOException {
- MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader();
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
}
@Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
throws IOException {
- MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader();
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
}
- private DimensionColumnChunkReader getDimensionColumnChunkReader() throws IOException {
+ private DimensionColumnChunkReader getDimensionColumnChunkReader(FileHolder fileReader) {
ColumnarFormatVersion version =
ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
- return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(
- version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(),
- dimensionLens,
- blockInfos.get(index).getFilePath());
+ if (fileReader.isReadPageByPage()) {
+ return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
+ blockInfos.get(index).getFilePath(), true);
+ } else {
+ return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
+ blockInfos.get(index).getFilePath(), false);
+ }
}
- private MeasureColumnChunkReader getMeasureColumnChunkReader() throws IOException {
+ private MeasureColumnChunkReader getMeasureColumnChunkReader(FileHolder fileReader) {
ColumnarFormatVersion version =
ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
- return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(),
- blockInfos.get(index).getFilePath());
+ if (fileReader.isReadPageByPage()) {
+ return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(),
+ blockInfos.get(index).getFilePath(), true);
+ } else {
+ return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(),
+ blockInfos.get(index).getFilePath(), false);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 694f8ae..c33d5ac 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -315,6 +315,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// total number dimension
blockExecutionInfo
.setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
+ blockExecutionInfo.setPrefetchBlocklet(!queryModel.isReadPageByPage());
blockExecutionInfo
.setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index adb6dc6..65f2838 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -223,6 +223,11 @@ public class BlockExecutionInfo {
*/
private String[] deleteDeltaFilePath;
+ /**
+ * whether to prefetch the blocklet data while scanning
+ */
+ private boolean prefetchBlocklet = true;
+
private Map<String, DeleteDeltaVo> deletedRecordsMap;
public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
return absoluteTableIdentifier;
@@ -658,4 +663,12 @@ public class BlockExecutionInfo {
public void setDeletedRecordsMap(Map<String, DeleteDeltaVo> deletedRecordsMap) {
this.deletedRecordsMap = deletedRecordsMap;
}
+
+ public boolean isPrefetchBlocklet() {
+ return prefetchBlocklet;
+ }
+
+ public void setPrefetchBlocklet(boolean prefetchBlocklet) {
+ this.prefetchBlocklet = prefetchBlocklet;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 5e4872b..655ed5e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -112,6 +112,11 @@ public class QueryModel implements Serializable {
private boolean[] isFilterDimensions;
private boolean[] isFilterMeasures;
+ /**
+ * Read the data from carbondata file page by page instead of whole blocklet.
+ */
+ private boolean readPageByPage;
+
public QueryModel() {
tableBlockInfos = new ArrayList<TableBlockInfo>();
queryDimension = new ArrayList<QueryDimension>();
@@ -417,4 +422,12 @@ public class QueryModel implements Serializable {
public void setIsFilterMeasures(boolean[] isFilterMeasures) {
this.isFilterMeasures = isFilterMeasures;
}
+
+ public boolean isReadPageByPage() {
+ return readPageByPage;
+ }
+
+ public void setReadPageByPage(boolean readPageByPage) {
+ this.readPageByPage = readPageByPage;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
index 798d331..eb5e3f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
@@ -139,15 +139,24 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
private AbstractScannedResult getNextScannedResult() throws Exception {
AbstractScannedResult result = null;
- if (dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get()) {
- if (future == null) {
- future = execute();
+ if (blockExecutionInfo.isPrefetchBlocklet()) {
+ if (dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get()) {
+ if (future == null) {
+ future = execute();
+ }
+ result = future.get();
+ nextBlock.set(false);
+ if (dataBlockIterator.hasNext() || nextRead.get()) {
+ nextBlock.set(true);
+ future = execute();
+ }
}
- result = future.get();
- nextBlock.set(false);
- if (dataBlockIterator.hasNext() || nextRead.get()) {
- nextBlock.set(true);
- future = execute();
+ } else {
+ if (dataBlockIterator.hasNext()) {
+ BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolder();
+ if (blocksChunkHolder != null) {
+ result = blockletScanner.scanBlocklet(blocksChunkHolder);
+ }
}
}
return result;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
index 7cc02ad..b089fad 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
@@ -83,7 +84,12 @@ public abstract class AbstractScannedResult {
/**
* Raw dimension chunks;
*/
- protected DimensionRawColumnChunk[] rawColumnChunks;
+ protected DimensionRawColumnChunk[] dimRawColumnChunks;
+
+ /**
+ * Raw dimension chunks;
+ */
+ protected MeasureRawColumnChunk[] msrRawColumnChunks;
/**
* measure column data chunk
*/
@@ -172,8 +178,12 @@ public abstract class AbstractScannedResult {
this.measureDataChunks = measureDataChunks;
}
- public void setRawColumnChunks(DimensionRawColumnChunk[] rawColumnChunks) {
- this.rawColumnChunks = rawColumnChunks;
+ public void setDimRawColumnChunks(DimensionRawColumnChunk[] dimRawColumnChunks) {
+ this.dimRawColumnChunks = dimRawColumnChunks;
+ }
+
+ public void setMsrRawColumnChunks(MeasureRawColumnChunk[] msrRawColumnChunks) {
+ this.msrRawColumnChunks = msrRawColumnChunks;
}
/**
@@ -269,7 +279,7 @@ public abstract class AbstractScannedResult {
DataOutputStream dataOutput = new DataOutputStream(byteStream);
try {
vectorInfos[i].genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(rawColumnChunks,
+ .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks,
rowMapping == null ? j : rowMapping[pageCounter][j], pageCounter, dataOutput);
Object data = vectorInfos[i].genericQueryType
.getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
@@ -325,11 +335,51 @@ public abstract class AbstractScannedResult {
rowCounter = 0;
currentRow = -1;
pageCounter++;
+ fillDataChunks();
if (null != deletedRecordMap) {
currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
}
}
+ /**
+ * This case is used only in case of compaction, since it does not use filter flow.
+ */
+ public void fillDataChunks() {
+ freeDataChunkMemory();
+ if (pageCounter >= numberOfRows.length) {
+ return;
+ }
+ for (int i = 0; i < dimensionDataChunks.length; i++) {
+ if (dimensionDataChunks[i][pageCounter] == null && dimRawColumnChunks[i] != null) {
+ dimensionDataChunks[i][pageCounter] =
+ dimRawColumnChunks[i].convertToDimColDataChunkWithOutCache(pageCounter);
+ }
+ }
+
+ for (int i = 0; i < measureDataChunks.length; i++) {
+ if (measureDataChunks[i][pageCounter] == null && msrRawColumnChunks[i] != null) {
+ measureDataChunks[i][pageCounter] =
+ msrRawColumnChunks[i].convertToColumnPageWithOutCache(pageCounter);
+ }
+ }
+ }
+
+ // free the memory for the last page chunk
+ private void freeDataChunkMemory() {
+ for (int i = 0; i < dimensionDataChunks.length; i++) {
+ if (pageCounter > 0 && dimensionDataChunks[i][pageCounter - 1] != null) {
+ dimensionDataChunks[i][pageCounter - 1].freeMemory();
+ dimensionDataChunks[i][pageCounter - 1] = null;
+ }
+ }
+ for (int i = 0; i < measureDataChunks.length; i++) {
+ if (pageCounter > 0 && measureDataChunks[i][pageCounter - 1] != null) {
+ measureDataChunks[i][pageCounter - 1].freeMemory();
+ measureDataChunks[i][pageCounter - 1] = null;
+ }
+ }
+ }
+
public int numberOfpages() {
return numberOfRows.length;
}
@@ -451,7 +501,7 @@ public abstract class AbstractScannedResult {
DataOutputStream dataOutput = new DataOutputStream(byteStream);
try {
genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(rawColumnChunks, rowId, pageCounter,
+ .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, rowId, pageCounter,
dataOutput);
complexTypeData[i] = byteStream.toByteArray();
} catch (IOException e) {
@@ -481,6 +531,7 @@ public abstract class AbstractScannedResult {
return true;
} else if (pageCounter < numberOfRows.length) {
pageCounter++;
+ fillDataChunks();
rowCounter = 0;
currentRow = -1;
if (null != deletedRecordMap) {
@@ -520,10 +571,10 @@ public abstract class AbstractScannedResult {
}
}
// free the raw chunks
- if (null != rawColumnChunks) {
- for (int i = 0; i < rawColumnChunks.length; i++) {
- if (null != rawColumnChunks[i]) {
- rawColumnChunks[i].freeMemory();
+ if (null != dimRawColumnChunks) {
+ for (int i = 0; i < dimRawColumnChunks.length; i++) {
+ if (null != dimRawColumnChunks[i]) {
+ dimRawColumnChunks[i].freeMemory();
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 8dd61b1..6172b40 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -108,6 +108,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
this.fileReader = FileFactory.getFileHolder(
FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getTablePath()));
this.fileReader.setQueryId(queryModel.getQueryId());
+ this.fileReader.setReadPageByPage(queryModel.isReadPageByPage());
this.execService = execService;
intialiseInfos();
initQueryStatiticsModel();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
index 19d6f48..bf26ca3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
@@ -77,25 +77,34 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
scannedResult.setBlockletId(
blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
.getDataBlock().blockletId());
+ if (!blockExecutionInfo.isPrefetchBlocklet()) {
+ readBlocklet(blocksChunkHolder);
+ }
DimensionRawColumnChunk[] dimensionRawColumnChunks =
blocksChunkHolder.getDimensionRawDataChunk();
DimensionColumnDataChunk[][] dimensionColumnDataChunks =
- new DimensionColumnDataChunk[dimensionRawColumnChunks.length][];
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (dimensionRawColumnChunks[i] != null) {
- dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
- }
- }
- scannedResult.setDimensionChunks(dimensionColumnDataChunks);
+ new DimensionColumnDataChunk[dimensionRawColumnChunks.length][blocksChunkHolder
+ .getDataBlock().numberOfPages()];
MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getMeasureRawDataChunk();
ColumnPage[][] columnPages =
- new ColumnPage[measureRawColumnChunks.length][];
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (measureRawColumnChunks[i] != null) {
- columnPages[i] = measureRawColumnChunks[i].convertToColumnPage();
+ new ColumnPage[measureRawColumnChunks.length][blocksChunkHolder.getDataBlock()
+ .numberOfPages()];
+ scannedResult.setDimensionChunks(dimensionColumnDataChunks);
+ scannedResult.setMeasureChunks(columnPages);
+ scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
+ if (blockExecutionInfo.isPrefetchBlocklet()) {
+ for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
+ if (dimensionRawColumnChunks[i] != null) {
+ dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
+ }
+ }
+ for (int i = 0; i < measureRawColumnChunks.length; i++) {
+ if (measureRawColumnChunks[i] != null) {
+ columnPages[i] = measureRawColumnChunks[i].convertToColumnPage();
+ }
}
}
- scannedResult.setMeasureChunks(columnPages);
int[] numberOfRows = null;
if (blockExecutionInfo.getAllSelectedDimensionBlocksIndexes().length > 0) {
for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
@@ -112,6 +121,7 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
}
}
}
+
// count(*) case there would not be any dimensions are measures selected.
if (numberOfRows == null) {
numberOfRows = new int[blocksChunkHolder.getDataBlock().numberOfPages()];
@@ -127,7 +137,9 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
}
}
scannedResult.setNumberOfRows(numberOfRows);
- scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
+ if (!blockExecutionInfo.isPrefetchBlocklet()) {
+ scannedResult.fillDataChunks();
+ }
// adding statistics for carbon scan time
QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
.get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index 79f9b49..e77093b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -309,7 +309,8 @@ public class FilterScanner extends AbstractBlockletScanner {
scannedResult.setDimensionChunks(dimensionColumnDataChunks);
scannedResult.setIndexes(indexesGroup);
scannedResult.setMeasureChunks(columnPages);
- scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
scannedResult.setNumberOfRows(rowCount);
// adding statistics for carbon scan time
QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index b1c0c30..5d7a09f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -24,6 +24,7 @@ import java.io.Closeable;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.UnsupportedEncodingException;
@@ -1467,6 +1468,22 @@ public final class CarbonUtil {
}, offset, length);
}
+ public static DataChunk3 readDataChunk3(InputStream stream) throws IOException {
+ TBaseCreator creator = new ThriftReader.TBaseCreator() {
+ @Override public TBase create() {
+ return new DataChunk3();
+ }
+ };
+ TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
+ TBase t = creator.create();
+ try {
+ t.read(binaryIn);
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ return (DataChunk3) t;
+ }
+
public static DataChunk2 readDataChunk(ByteBuffer dataChunkBuffer, int offset, int length)
throws IOException {
byte[] data = dataChunkBuffer.array();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
index 65f70a0..f51ced3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
@@ -45,6 +45,7 @@ import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.result.BatchResult;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
+import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -205,6 +206,7 @@ public class CarbonCompactionExecutor {
model.setForcedDetailRawQuery(true);
model.setFilterExpressionResolverTree(null);
model.setConverter(DataTypeUtil.getDataTypeConverter());
+ model.setReadPageByPage(enablePageLevelReaderForCompaction());
List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
@@ -234,4 +236,21 @@ public class CarbonCompactionExecutor {
return model;
}
+ /**
+ * Whether to enable page level reader for compaction or not.
+ */
+ private boolean enablePageLevelReaderForCompaction() {
+ String enablePageReaderProperty = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION,
+ CarbonCommonConstants.CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION_DEFAULT);
+ boolean enablePageReader;
+ try {
+ enablePageReader = Boolean.parseBoolean(enablePageReaderProperty);
+ } catch (Exception e) {
+ enablePageReader = Boolean.parseBoolean(
+ CarbonCommonConstants.CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION_DEFAULT);
+ }
+ return enablePageReader;
+ }
+
}
[25/50] [abbrv] carbondata git commit: [CARBONDATA-1992] Remove
partitionId in CarbonTablePath
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
index 8b87cfc..6cf1dcd 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
@@ -88,11 +89,13 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
child.initialize();
}
- private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) {
- String[] storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
- tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId,
- configuration.getSegmentId() + "", false, false);
+ private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) {
+ String[] storeLocation = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ tableIdentifier.getDatabaseName(),
+ tableIdentifier.getTableName(),
+ String.valueOf(configuration.getTaskNo()), configuration.getSegmentId(),
+ false,
+ false);
CarbonDataProcessorUtil.createLocations(storeLocation);
return storeLocation;
}
@@ -115,11 +118,11 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
measureCount = configuration.getMeasureCount();
outputLength = measureCount + (this.noDictWithComplextCount > 0 ? 1 : 0) + 1;
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
if (iterators.length == 1) {
- doExecute(iterators[0], 0, 0);
+ doExecute(iterators[0], 0);
} else {
executorService = Executors.newFixedThreadPool(iterators.length,
new CarbonThreadFactory("NoSortDataWriterPool:" + configuration.getTableIdentifier()
@@ -150,11 +153,10 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
return null;
}
- private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) {
- String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
- CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
- .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId,
- iteratorIndex);
+ private void doExecute(Iterator<CarbonRowBatch> iterator, int iteratorIndex) {
+ String[] storeLocation = getStoreLocation(tableIdentifier);
+ CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(
+ configuration, storeLocation, 0, iteratorIndex);
CarbonFactHandler dataHandler = null;
boolean rowsNotExist = true;
while (iterator.hasNext()) {
@@ -189,10 +191,11 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
processingComplete(dataHandler);
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis());
+ .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID,
+ System.currentTimeMillis());
}
private void processingComplete(CarbonFactHandler dataHandler) throws CarbonDataLoadingException {
@@ -298,7 +301,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
}
@Override public void run() {
- doExecute(this.iterator, 0, iteratorIndex);
+ doExecute(this.iterator, iteratorIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
index f030d52..369c1f2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
@@ -59,13 +60,11 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
child.initialize();
}
- private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) {
- String[] storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
- tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId,
- configuration.getSegmentId() + "", false, false);
- CarbonDataProcessorUtil.createLocations(storeLocation);
- return storeLocation;
+ private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) {
+ return CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ tableIdentifier.getDatabaseName(), tableIdentifier.getTableName(),
+ String.valueOf(configuration.getTaskNo()),
+ configuration.getSegmentId(), false, false);
}
@Override public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
@@ -75,18 +74,19 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
String tableName = tableIdentifier.getTableName();
try {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
int i = 0;
+ String[] storeLocation = getStoreLocation(tableIdentifier);
+ CarbonDataProcessorUtil.createLocations(storeLocation);
for (Iterator<CarbonRowBatch> iterator : iterators) {
- String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(i));
int k = 0;
while (iterator.hasNext()) {
CarbonRowBatch next = iterator.next();
// If no rows from merge sorter, then don't create a file in fact column handler
if (next.hasNext()) {
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
- .createCarbonFactDataHandlerModel(configuration, storeLocation, i, k++);
+ .createCarbonFactDataHandlerModel(configuration, storeLocation, 0, k++);
CarbonFactHandler dataHandler = CarbonFactHandlerFactory
.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
dataHandler.initialise();
@@ -119,10 +119,11 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
processingComplete(dataHandler);
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis());
+ .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID,
+ System.currentTimeMillis());
}
private void processingComplete(CarbonFactHandler dataHandler) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
index d321405..58009af 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
@@ -65,21 +66,21 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
child.initialize();
}
- private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) {
+ private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) {
String[] storeLocation = CarbonDataProcessorUtil
.getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
- tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId,
- configuration.getSegmentId() + "", false, false);
+ tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()),
+ configuration.getSegmentId(), false, false);
CarbonDataProcessorUtil.createLocations(storeLocation);
return storeLocation;
}
- public CarbonFactDataHandlerModel getDataHandlerModel(int partitionId) {
+ public CarbonFactDataHandlerModel getDataHandlerModel() {
CarbonTableIdentifier tableIdentifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
- String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
+ String[] storeLocation = getStoreLocation(tableIdentifier);
return CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration,
- storeLocation, partitionId, 0);
+ storeLocation, 0, 0);
}
@Override public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
@@ -89,11 +90,11 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
String tableName = tableIdentifier.getTableName();
try {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
int i = 0;
for (Iterator<CarbonRowBatch> iterator : iterators) {
- String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(i));
+ String[] storeLocation = getStoreLocation(tableIdentifier);
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
.createCarbonFactDataHandlerModel(configuration, storeLocation, i, 0);
@@ -147,10 +148,11 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
processingComplete(dataHandler);
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis());
+ .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID,
+ System.currentTimeMillis());
}
private void processingComplete(CarbonFactHandler dataHandler) throws CarbonDataLoadingException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index be27866..0eadc7f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -319,7 +319,7 @@ public final class CarbonDataMergerUtil {
// create entry for merged one.
LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails();
- loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId());
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID);
loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS);
long loadEnddate = CarbonUpdateUtil.readCurrentTime();
loadMetadataDetails.setLoadEndTime(loadEnddate);
@@ -676,7 +676,7 @@ public final class CarbonDataMergerUtil {
CarbonTableIdentifier carbonTableIdentifier, String segmentId) {
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(tablePath, carbonTableIdentifier);
- return carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ return carbonTablePath.getCarbonDataDirectoryPath(segmentId);
}
@@ -1036,7 +1036,7 @@ public final class CarbonDataMergerUtil {
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", seg);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(seg);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 2480a39..ff65db2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -372,7 +372,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
return SortParameters
.createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,
dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,
- noDictionaryCount, carbonLoadModel.getPartitionId(), segmentId,
+ noDictionaryCount, segmentId,
carbonLoadModel.getTaskNo(), noDictionaryColMapping, true);
}
@@ -422,7 +422,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
private void initTempStoreLocation() {
tempStoreLocation = CarbonDataProcessorUtil
.getLocalDataFolderLocation(carbonLoadModel.getDatabaseName(), tableName,
- carbonLoadModel.getTaskNo(), carbonLoadModel.getPartitionId(), segmentId,
+ carbonLoadModel.getTaskNo(), segmentId,
true, false);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
index a2248ee..0c0b2b0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -356,7 +357,7 @@ public class SortParameters implements Serializable {
CarbonProperties carbonProperties = CarbonProperties.getInstance();
parameters.setDatabaseName(tableIdentifier.getDatabaseName());
parameters.setTableName(tableIdentifier.getTableName());
- parameters.setPartitionID(configuration.getPartitionId());
+ parameters.setPartitionID("0");
parameters.setSegmentId(configuration.getSegmentId());
parameters.setTaskNo(configuration.getTaskNo());
parameters.setMeasureColCount(configuration.getMeasureCount());
@@ -392,10 +393,9 @@ public class SortParameters implements Serializable {
LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize());
- String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
- tableIdentifier.getTableName(), configuration.getTaskNo(),
- configuration.getPartitionId(), configuration.getSegmentId(), false, false);
+ String[] carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ tableIdentifier.getDatabaseName(), tableIdentifier.getTableName(),
+ configuration.getTaskNo(), configuration.getSegmentId(), false, false);
String[] sortTempDirs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
@@ -444,13 +444,13 @@ public class SortParameters implements Serializable {
public static SortParameters createSortParameters(CarbonTable carbonTable, String databaseName,
String tableName, int dimColCount, int complexDimColCount, int measureColCount,
- int noDictionaryCount, String partitionID, String segmentId, String taskNo,
+ int noDictionaryCount, String segmentId, String taskNo,
boolean[] noDictionaryColMaping, boolean isCompactionFlow) {
SortParameters parameters = new SortParameters();
CarbonProperties carbonProperties = CarbonProperties.getInstance();
parameters.setDatabaseName(databaseName);
parameters.setTableName(tableName);
- parameters.setPartitionID(partitionID);
+ parameters.setPartitionID(CarbonTablePath.DEPRECATED_PATITION_ID);
parameters.setSegmentId(segmentId);
parameters.setTaskNo(taskNo);
parameters.setMeasureColCount(measureColCount);
@@ -482,7 +482,7 @@ public class SortParameters implements Serializable {
LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize());
String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId,
+ .getLocalDataFolderLocation(databaseName, tableName, taskNo, segmentId,
isCompactionFlow, false);
String[] sortTempDirs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index d15152c..9f3c86f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -309,7 +309,7 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes);
String carbonDataDirectoryPath = CarbonDataProcessorUtil
.checkAndCreateCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(),
- tableName, loadModel.getPartitionId(), loadModel.getSegmentId());
+ tableName, loadModel.getSegmentId());
carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
List<CarbonDimension> dimensionByTableName = carbonTable.getDimensionByTableName(tableName);
boolean[] isUseInvertedIndexes = new boolean[dimensionByTableName.size()];
@@ -336,9 +336,8 @@ public class CarbonFactDataHandlerModel {
private static String getCarbonDataFolderLocation(CarbonDataLoadConfiguration configuration) {
AbsoluteTableIdentifier absoluteTableIdentifier = configuration.getTableIdentifier();
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- String carbonDataDirectoryPath = carbonTablePath
- .getCarbonDataDirectoryPath(configuration.getPartitionId(),
- configuration.getSegmentId() + "");
+ String carbonDataDirectoryPath =
+ carbonTablePath.getCarbonDataDirectoryPath(configuration.getSegmentId());
CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
return carbonDataDirectoryPath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 2a4cc00..cfe6e31 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -123,13 +123,11 @@ public final class CarbonDataProcessorUtil {
* @param databaseName
* @param tableName
* @param taskId
- * @param partitionId
* @param segmentId
* @return
*/
public static String[] getLocalDataFolderLocation(String databaseName, String tableName,
- String taskId, String partitionId, String segmentId, boolean isCompactionFlow,
- boolean isAltPartitionFlow) {
+ String taskId, String segmentId, boolean isCompactionFlow, boolean isAltPartitionFlow) {
String tempLocationKey =
getTempStoreLocationKey(databaseName, tableName, segmentId, taskId, isCompactionFlow,
isAltPartitionFlow);
@@ -150,8 +148,7 @@ public final class CarbonDataProcessorUtil {
String tmpStore = baseTmpStorePathArray[i];
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(tmpStore, carbonTable.getCarbonTableIdentifier());
- String carbonDataDirectoryPath =
- carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId + "");
+ String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
localDataFolderLocArray[i] = carbonDataDirectoryPath + File.separator + taskId;
}
@@ -378,13 +375,12 @@ public final class CarbonDataProcessorUtil {
* @return data directory path
*/
public static String checkAndCreateCarbonStoreLocation(String factStoreLocation,
- String databaseName, String tableName, String partitionId, String segmentId) {
+ String databaseName, String tableName, String segmentId) {
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName);
CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(factStoreLocation, carbonTableIdentifier);
- String carbonDataDirectoryPath =
- carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId);
+ String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
return carbonDataDirectoryPath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 12fc5c1..7be61d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -77,10 +77,8 @@ public final class CarbonLoaderUtil {
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(loadModel.getTablePath(), carbonTable.getCarbonTableIdentifier());
- for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "", currentLoad + "");
- deleteStorePath(segmentPath);
- }
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(currentLoad + "");
+ deleteStorePath(segmentPath);
}
/**
@@ -100,7 +98,7 @@ public final class CarbonLoaderUtil {
int fileCount = 0;
int partitionCount = carbonTable.getPartitionCount();
for (int i = 0; i < partitionCount; i++) {
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "",
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(
currentLoad + "");
CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath,
FileFactory.getFileType(segmentPath));
@@ -274,7 +272,7 @@ public final class CarbonLoaderUtil {
private static void addToStaleFolders(CarbonTablePath carbonTablePath,
List<CarbonFile> staleFolders, LoadMetadataDetails entry) throws IOException {
- String path = carbonTablePath.getCarbonDataDirectoryPath("0", entry.getLoadName());
+ String path = carbonTablePath.getCarbonDataDirectoryPath(entry.getLoadName());
// add to the deletion list only if file exist else HDFS file system will throw
// exception while deleting the file if file path does not exist
if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
@@ -843,7 +841,7 @@ public final class CarbonLoaderUtil {
CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath(), carbonTableIdentifier);
- String segmentFolder = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentFolder = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonUtil.checkAndCreateFolder(segmentFolder);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
index 02ab1d8..f9f3e20 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
@@ -48,15 +48,14 @@ public final class DeleteLoadFolders {
* returns segment path
*
* @param absoluteTableIdentifier
- * @param partitionId
* @param oneLoad
* @return
*/
private static String getSegmentPath(AbsoluteTableIdentifier absoluteTableIdentifier,
- int partitionId, LoadMetadataDetails oneLoad) {
+ LoadMetadataDetails oneLoad) {
CarbonTablePath carbon = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
String segmentId = oneLoad.getLoadName();
- return carbon.getCarbonDataDirectoryPath("" + partitionId, segmentId);
+ return carbon.getCarbonDataDirectoryPath(segmentId);
}
public static void physicalFactAndMeasureMetadataDeletion(
@@ -64,7 +63,7 @@ public final class DeleteLoadFolders {
LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
for (LoadMetadataDetails oneLoad : currentDetails) {
if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) {
- String path = getSegmentPath(absoluteTableIdentifier, 0, oneLoad);
+ String path = getSegmentPath(absoluteTableIdentifier, oneLoad);
boolean status = false;
try {
if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
index e662757..7f0aef6 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
@@ -163,7 +163,6 @@ public class StoreCreator {
loadModel.setCsvHeaderColumns(loadModel.getCsvHeader().split(","));
loadModel.setTaskNo("0");
loadModel.setSegmentId("0");
- loadModel.setPartitionId("0");
loadModel.setFactTimeStamp(System.currentTimeMillis());
loadModel.setMaxColumns("10");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index a96ab32..36a5a15 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -105,7 +105,6 @@ class StreamHandoffRDD[K, V](
split: Partition,
context: TaskContext
): Iterator[(K, V)] = {
- carbonLoadModel.setPartitionId("0")
carbonLoadModel.setTaskNo("" + split.index)
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
CarbonMetadata.getInstance().addCarbonTable(carbonTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index 1c7be6a..f2274be 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -135,7 +135,7 @@ object StreamSinkFactory {
FileFactory.mkdirs(carbonTablePath.getMetadataDirectoryPath, fileType)
}
val segmentId = StreamSegment.open(carbonTable)
- val segmentDir = carbonTablePath.getSegmentDir("0", segmentId)
+ val segmentDir = carbonTablePath.getSegmentDir(segmentId)
if (FileFactory.isFileExist(segmentDir, fileType)) {
// recover fault
StreamSegment.recoverSegmentIfRequired(segmentDir)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/952665a8/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index 849bf99..45bc19a 100644
--- a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++ b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -149,12 +149,12 @@ class CarbonAppendableStreamSink(
* if the directory size of current segment beyond the threshold, hand off new segment
*/
private def checkOrHandOffSegment(): Unit = {
- val segmentDir = carbonTablePath.getSegmentDir("0", currentSegmentId)
+ val segmentDir = carbonTablePath.getSegmentDir(currentSegmentId)
val fileType = FileFactory.getFileType(segmentDir)
if (segmentMaxSize <= StreamSegment.size(segmentDir)) {
val newSegmentId = StreamSegment.close(carbonTable, currentSegmentId)
currentSegmentId = newSegmentId
- val newSegmentDir = carbonTablePath.getSegmentDir("0", currentSegmentId)
+ val newSegmentDir = carbonTablePath.getSegmentDir(currentSegmentId)
FileFactory.mkdirs(newSegmentDir, fileType)
// TODO trigger hand off operation
@@ -251,14 +251,14 @@ object CarbonAppendableStreamSink {
// update data file info in index file
val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- StreamSegment.updateIndexFile(tablePath.getSegmentDir("0", segmentId))
+ StreamSegment.updateIndexFile(tablePath.getSegmentDir(segmentId))
} catch {
// catch fault of executor side
case t: Throwable =>
val tablePath =
CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = tablePath.getSegmentDir("0", segmentId)
+ val segmentDir = tablePath.getSegmentDir(segmentId)
StreamSegment.recoverSegmentIfRequired(segmentDir)
LOGGER.error(t, s"Aborting job ${ job.getJobID }.")
committer.abortJob(job)
[33/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
index 553f85e..773fbd7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
@@ -17,20 +17,15 @@
package org.apache.carbondata.core.scan.result.iterator;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
public class PartitionSpliterRawResultIterator extends CarbonIterator<Object[]> {
- private CarbonIterator<BatchResult> iterator;
- private BatchResult batch;
+ private CarbonIterator<RowBatch> iterator;
+ private RowBatch batch;
private int counter;
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(PartitionSpliterRawResultIterator.class.getName());
-
- public PartitionSpliterRawResultIterator(CarbonIterator<BatchResult> iterator) {
+ public PartitionSpliterRawResultIterator(CarbonIterator<RowBatch> iterator) {
this.iterator = iterator;
}
@@ -65,7 +60,7 @@ public class PartitionSpliterRawResultIterator extends CarbonIterator<Object[]>
* @param batch
* @return
*/
- private boolean checkBatchEnd(BatchResult batch) {
+ private boolean checkBatchEnd(RowBatch batch) {
return !(counter < batch.getSize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
index 70d0958..1dd1595 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
@@ -21,7 +21,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
/**
@@ -37,7 +37,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
/**
* Iterator of the Batch raw result.
*/
- private CarbonIterator<BatchResult> detailRawQueryResultIterator;
+ private CarbonIterator<RowBatch> detailRawQueryResultIterator;
/**
* Counter to maintain the row counter.
@@ -55,9 +55,9 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
/**
* batch of the result.
*/
- private BatchResult batch;
+ private RowBatch batch;
- public RawResultIterator(CarbonIterator<BatchResult> detailRawQueryResultIterator,
+ public RawResultIterator(CarbonIterator<RowBatch> detailRawQueryResultIterator,
SegmentProperties sourceSegProperties, SegmentProperties destinationSegProperties) {
this.detailRawQueryResultIterator = detailRawQueryResultIterator;
this.sourceSegProperties = sourceSegProperties;
@@ -155,7 +155,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
* @param batch
* @return
*/
- private boolean checkIfBatchIsProcessedCompletely(BatchResult batch) {
+ private boolean checkIfBatchIsProcessedCompletely(RowBatch batch) {
if (counter < batch.getSize()) {
return false;
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
index cc9710e..c7cb00d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
@@ -35,10 +35,12 @@ public class VectorDetailQueryResultIterator extends AbstractDetailQueryResultIt
super(infos, queryModel, execService);
}
- @Override public Object next() {
+ @Override
+ public Object next() {
throw new UnsupportedOperationException("call processNextBatch instead");
}
+ @Override
public void processNextBatch(CarbonColumnarBatch columnarBatch) {
synchronized (lock) {
updateDataBlockIterator();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
index cfc2f16..973ce0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
@@ -87,7 +87,4 @@ public class CarbonColumnarBatch {
}
}
- public int getRowsFilteredCount() {
- return rowsFiltered;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
index a5f81b9..59117dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
@@ -18,16 +18,16 @@ package org.apache.carbondata.core.scan.result.vector;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
public class ColumnVectorInfo implements Comparable<ColumnVectorInfo> {
public int offset;
public int size;
public CarbonColumnVector vector;
public int vectorOffset;
- public QueryDimension dimension;
- public QueryMeasure measure;
+ public ProjectionDimension dimension;
+ public ProjectionMeasure measure;
public int ordinal;
public DirectDictionaryGenerator directDictionaryGenerator;
public MeasureDataVectorProcessor.MeasureVectorFiller measureVectorFiller;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
index db4c982..8902dfb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
@@ -29,7 +29,7 @@ public class MeasureDataVectorProcessor {
void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info);
- void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info);
}
@@ -60,7 +60,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -69,13 +69,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putInt(vectorOffset, (int)dataChunk.getLong(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -117,7 +117,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping,
+ public void fillMeasureVector(int[] filteredRowId,
ColumnPage dataChunk, ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -126,13 +126,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putBoolean(vectorOffset, dataChunk.getBoolean(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -171,7 +171,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -180,13 +180,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putShort(vectorOffset, (short) dataChunk.getLong(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -225,7 +225,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -234,13 +234,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putLong(vectorOffset, dataChunk.getLong(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -279,7 +279,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -288,7 +288,7 @@ public class MeasureDataVectorProcessor {
int precision = info.measure.getMeasure().getPrecision();
BitSet nullBitSet = dataChunk.getNullBits();
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -330,7 +330,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -339,13 +339,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putDouble(vectorOffset, dataChunk.getDouble(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
deleted file mode 100644
index bf26ca3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.scanner;
-
-import java.io.IOException;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
-import org.apache.carbondata.core.scan.result.impl.NonFilterQueryScannedResult;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-
-/**
- * Blocklet scanner class to process the block
- */
-public abstract class AbstractBlockletScanner implements BlockletScanner {
-
- /**
- * block execution info
- */
- protected BlockExecutionInfo blockExecutionInfo;
-
- public QueryStatisticsModel queryStatisticsModel;
-
- private AbstractScannedResult emptyResult;
-
- public AbstractBlockletScanner(BlockExecutionInfo tableBlockExecutionInfos) {
- this.blockExecutionInfo = tableBlockExecutionInfos;
- }
-
- @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws IOException, FilterUnsupportedException {
- long startTime = System.currentTimeMillis();
- AbstractScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
- QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
- totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
- totalBlockletStatistic.getCount() + 1);
- QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
- validScannedBlockletStatistic
- .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
- validScannedBlockletStatistic.getCount() + 1);
- // adding statistics for valid number of pages
- QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
- validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
- validPages.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
- // adding statistics for number of pages
- QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
- totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
- totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
- scannedResult.setBlockletId(
- blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
- .getDataBlock().blockletId());
- if (!blockExecutionInfo.isPrefetchBlocklet()) {
- readBlocklet(blocksChunkHolder);
- }
- DimensionRawColumnChunk[] dimensionRawColumnChunks =
- blocksChunkHolder.getDimensionRawDataChunk();
- DimensionColumnDataChunk[][] dimensionColumnDataChunks =
- new DimensionColumnDataChunk[dimensionRawColumnChunks.length][blocksChunkHolder
- .getDataBlock().numberOfPages()];
- MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getMeasureRawDataChunk();
- ColumnPage[][] columnPages =
- new ColumnPage[measureRawColumnChunks.length][blocksChunkHolder.getDataBlock()
- .numberOfPages()];
- scannedResult.setDimensionChunks(dimensionColumnDataChunks);
- scannedResult.setMeasureChunks(columnPages);
- scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
- scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
- if (blockExecutionInfo.isPrefetchBlocklet()) {
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (dimensionRawColumnChunks[i] != null) {
- dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
- }
- }
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (measureRawColumnChunks[i] != null) {
- columnPages[i] = measureRawColumnChunks[i].convertToColumnPage();
- }
- }
- }
- int[] numberOfRows = null;
- if (blockExecutionInfo.getAllSelectedDimensionBlocksIndexes().length > 0) {
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (dimensionRawColumnChunks[i] != null) {
- numberOfRows = dimensionRawColumnChunks[i].getRowCount();
- break;
- }
- }
- } else if (blockExecutionInfo.getAllSelectedMeasureBlocksIndexes().length > 0) {
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (measureRawColumnChunks[i] != null) {
- numberOfRows = measureRawColumnChunks[i].getRowCount();
- break;
- }
- }
- }
-
- // count(*) case there would not be any dimensions are measures selected.
- if (numberOfRows == null) {
- numberOfRows = new int[blocksChunkHolder.getDataBlock().numberOfPages()];
- for (int i = 0; i < numberOfRows.length; i++) {
- numberOfRows[i] =
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- }
- int lastPageSize = blocksChunkHolder.getDataBlock().nodeSize()
- % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- ;
- if (lastPageSize > 0) {
- numberOfRows[numberOfRows.length - 1] = lastPageSize;
- }
- }
- scannedResult.setNumberOfRows(numberOfRows);
- if (!blockExecutionInfo.isPrefetchBlocklet()) {
- scannedResult.fillDataChunks();
- }
- // adding statistics for carbon scan time
- QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
- scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
- scanTime.getCount() + (System.currentTimeMillis() - startTime));
- return scannedResult;
- }
-
- @Override public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
- long startTime = System.currentTimeMillis();
- DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDataBlock()
- .getDimensionChunks(blocksChunkHolder.getFileReader(),
- blockExecutionInfo.getAllSelectedDimensionBlocksIndexes());
- blocksChunkHolder.setDimensionRawDataChunk(dimensionRawColumnChunks);
- MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getDataBlock()
- .getMeasureChunks(blocksChunkHolder.getFileReader(),
- blockExecutionInfo.getAllSelectedMeasureBlocksIndexes());
- blocksChunkHolder.setMeasureRawDataChunk(measureRawColumnChunks);
- // adding statistics for carbon read time
- QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
- readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
- readTime.getCount() + (System.currentTimeMillis() - startTime));
- }
-
- @Override public AbstractScannedResult createEmptyResult() {
- if (emptyResult == null) {
- emptyResult = new NonFilterQueryScannedResult(blockExecutionInfo);
- emptyResult.setNumberOfRows(new int[0]);
- emptyResult.setIndexes(new int[0][]);
- }
- return emptyResult;
- }
-
- @Override public boolean isScanRequired(BlocksChunkHolder blocksChunkHolder) throws IOException {
- // For non filter it is always true
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
index 0ed0d43..0a41032 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
@@ -18,9 +18,10 @@ package org.apache.carbondata.core.scan.scanner;
import java.io.IOException;
+import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
/**
* Interface for processing the block
@@ -30,31 +31,26 @@ public interface BlockletScanner {
/**
* Checks whether this blocklet required to scan or not based on min max of each blocklet.
- * @param blocksChunkHolder
+ * @param dataBlock
* @return
* @throws IOException
*/
- boolean isScanRequired(BlocksChunkHolder blocksChunkHolder) throws IOException;
+ boolean isScanRequired(DataRefNode dataBlock);
/**
* Below method will used to process the block data and get the scanned result
*
- * @param blocksChunkHolder block chunk which holds the block data
+ * @param rawBlockletColumnChunks block chunk which holds the block data
* @return scannerResult
* result after processing
*/
- AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
+ BlockletScannedResult scanBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks)
throws IOException, FilterUnsupportedException;
/**
* Just reads the blocklet from file, does not uncompress it.
- * @param blocksChunkHolder
+ * @param rawBlockletColumnChunks
*/
- void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException;
+ void readBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException;
- /**
- * In case if there is no filter satisfies.
- * @return AbstractScannedResult
- */
- AbstractScannedResult createEmptyResult();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
new file mode 100644
index 0000000..1c73d63
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.scanner.impl;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
+import org.apache.carbondata.core.scan.result.impl.FilterQueryScannedResult;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.util.BitSetGroup;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * Below class will be used for filter query processing
+ * this class will be first apply the filter then it will read the column page if
+ * required and return the scanned result
+ */
+public class BlockletFilterScanner extends BlockletFullScanner {
+
+ /**
+ * filter executer to evaluate filter condition
+ */
+ private FilterExecuter filterExecuter;
+ /**
+ * this will be used to apply min max
+ * this will be useful for dimension column which is on the right side
+ * as node finder will always give tentative blocks, if column data stored individually
+ * and data is in sorted order then we can check whether filter is in the range of min max or not
+ * if it present then only we can apply filter on complete data.
+ * this will be very useful in case of sparse data when rows are
+ * repeating.
+ */
+ private boolean isMinMaxEnabled;
+
+ private QueryStatisticsModel queryStatisticsModel;
+
+ private boolean useBitSetPipeLine;
+
+ public BlockletFilterScanner(BlockExecutionInfo blockExecutionInfo,
+ QueryStatisticsModel queryStatisticsModel) {
+ super(blockExecutionInfo, queryStatisticsModel);
+ // to check whether min max is enabled or not
+ String minMaxEnableValue = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_QUERY_MIN_MAX_ENABLED,
+ CarbonCommonConstants.MIN_MAX_DEFAULT_VALUE);
+ if (null != minMaxEnableValue) {
+ isMinMaxEnabled = Boolean.parseBoolean(minMaxEnableValue);
+ }
+ // get the filter tree
+ this.filterExecuter = blockExecutionInfo.getFilterExecuterTree();
+ this.queryStatisticsModel = queryStatisticsModel;
+
+ String useBitSetPipeLine = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.BITSET_PIPE_LINE,
+ CarbonCommonConstants.BITSET_PIPE_LINE_DEFAULT);
+ if (null != useBitSetPipeLine) {
+ this.useBitSetPipeLine = Boolean.parseBoolean(useBitSetPipeLine);
+ }
+ }
+
+ /**
+ * Below method will be used to process the block
+ *
+ * @param rawBlockletColumnChunks block chunk holder which holds the data
+ * @throws FilterUnsupportedException
+ */
+ @Override
+ public BlockletScannedResult scanBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws IOException, FilterUnsupportedException {
+ return executeFilter(rawBlockletColumnChunks);
+ }
+
+ @Override
+ public boolean isScanRequired(DataRefNode dataBlock) {
+ // adding statistics for number of pages
+ QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
+ totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
+ totalPagesScanned.getCount() + dataBlock.numberOfPages());
+ // apply min max
+ if (isMinMaxEnabled) {
+ BitSet bitSet = null;
+ // check for implicit include filter instance
+ if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
+ String blockletId = blockExecutionInfo.getBlockIdString() +
+ CarbonCommonConstants.FILE_SEPARATOR + dataBlock.blockletIndex();
+ bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
+ .isFilterValuesPresentInBlockOrBlocklet(
+ dataBlock.getColumnsMaxValue(),
+ dataBlock.getColumnsMinValue(), blockletId);
+ } else {
+ bitSet = this.filterExecuter
+ .isScanRequired(dataBlock.getColumnsMaxValue(),
+ dataBlock.getColumnsMinValue());
+ }
+ return !bitSet.isEmpty();
+ }
+ return true;
+ }
+
+ @Override
+ public void readBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ long startTime = System.currentTimeMillis();
+ this.filterExecuter.readColumnChunks(rawBlockletColumnChunks);
+ // adding statistics for carbon read time
+ QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+ readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+ readTime.getCount() + (System.currentTimeMillis() - startTime));
+ }
+
+ /**
+ * This method will process the data in below order
+ * 1. first apply min max on the filter tree and check whether any of the filter
+ * is fall on the range of min max, if not then return empty result
+ * 2. If filter falls on min max range then apply filter on actual
+ * data and get the filtered row index
+ * 3. if row index is empty then return the empty result
+ * 4. if row indexes is not empty then read only those blocks(measure or dimension)
+ * which was present in the query but not present in the filter, as while applying filter
+ * some of the blocks where already read and present in chunk holder so not need to
+ * read those blocks again, this is to avoid reading of same blocks which was already read
+ * 5. Set the blocks and filter indexes to result
+ *
+ * @param rawBlockletColumnChunks
+ * @throws FilterUnsupportedException
+ */
+ private BlockletScannedResult executeFilter(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ long startTime = System.currentTimeMillis();
+ QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
+ totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
+ totalBlockletStatistic.getCount() + 1);
+ // apply filter on actual data, for each page
+ BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(rawBlockletColumnChunks,
+ useBitSetPipeLine);
+ // if filter result is empty then return with empty result
+ if (bitSetGroup.isEmpty()) {
+ CarbonUtil.freeMemory(rawBlockletColumnChunks.getDimensionRawColumnChunks(),
+ rawBlockletColumnChunks.getMeasureRawColumnChunks());
+
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime));
+
+ QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.PAGE_SCANNED);
+ scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
+ scannedPages.getCount() + bitSetGroup.getScannedPages());
+ return createEmptyResult();
+ }
+
+ BlockletScannedResult scannedResult = new FilterQueryScannedResult(blockExecutionInfo);
+ scannedResult.setBlockletId(
+ blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR +
+ rawBlockletColumnChunks.getDataBlock().blockletIndex());
+ // valid scanned blocklet
+ QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
+ validScannedBlockletStatistic
+ .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
+ validScannedBlockletStatistic.getCount() + 1);
+ // adding statistics for valid number of pages
+ QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
+ validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
+ validPages.getCount() + bitSetGroup.getValidPages());
+ QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.PAGE_SCANNED);
+ scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
+ scannedPages.getCount() + bitSetGroup.getScannedPages());
+ int[] pageFilteredRowCount = new int[bitSetGroup.getNumberOfPages()];
+ // get the row indexes from bit set for each page
+ int[][] pageFilteredRowId = new int[bitSetGroup.getNumberOfPages()][];
+ int numPages = pageFilteredRowId.length;
+ for (int pageId = 0; pageId < numPages; pageId++) {
+ BitSet bitSet = bitSetGroup.getBitSet(pageId);
+ if (bitSet != null && !bitSet.isEmpty()) {
+ int[] matchedRowId = new int[bitSet.cardinality()];
+ int index = 0;
+ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
+ matchedRowId[index++] = i;
+ }
+ pageFilteredRowCount[pageId] = matchedRowId.length;
+ pageFilteredRowId[pageId] = matchedRowId;
+ }
+ }
+
+ long dimensionReadTime = System.currentTimeMillis();
+ dimensionReadTime = System.currentTimeMillis() - dimensionReadTime;
+
+ FileReader fileReader = rawBlockletColumnChunks.getFileReader();
+
+
+ DimensionRawColumnChunk[] dimensionRawColumnChunks =
+ new DimensionRawColumnChunk[blockExecutionInfo.getTotalNumberDimensionToRead()];
+ int numDimensionChunks = dimensionRawColumnChunks.length;
+ // read dimension chunk blocks from file which is not present
+ for (int chunkIndex = 0; chunkIndex < numDimensionChunks; chunkIndex++) {
+ dimensionRawColumnChunks[chunkIndex] =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ }
+ int[][] allSelectedDimensionColumnIndexRange =
+ blockExecutionInfo.getAllSelectedDimensionColumnIndexRange();
+ DimensionRawColumnChunk[] projectionListDimensionChunk = rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunks(fileReader, allSelectedDimensionColumnIndexRange);
+ for (int[] columnIndexRange : allSelectedDimensionColumnIndexRange) {
+ System.arraycopy(projectionListDimensionChunk, columnIndexRange[0],
+ dimensionRawColumnChunks, columnIndexRange[0],
+ columnIndexRange[1] + 1 - columnIndexRange[0]);
+ }
+
+ /*
+ * in case projection if the projected dimension are not loaded in the dimensionColumnDataChunk
+ * then loading them
+ */
+ int[] projectionListDimensionIndexes = blockExecutionInfo.getProjectionListDimensionIndexes();
+ for (int projectionListDimensionIndex : projectionListDimensionIndexes) {
+ if (null == dimensionRawColumnChunks[projectionListDimensionIndex]) {
+ dimensionRawColumnChunks[projectionListDimensionIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ fileReader, projectionListDimensionIndex);
+ }
+ }
+
+ DimensionColumnPage[][] dimensionColumnPages =
+ new DimensionColumnPage[numDimensionChunks][numPages];
+ for (int chunkIndex = 0; chunkIndex < numDimensionChunks; chunkIndex++) {
+ if (dimensionRawColumnChunks[chunkIndex] != null) {
+ for (int pageId = 0; pageId < numPages; pageId++) {
+ dimensionColumnPages[chunkIndex][pageId] =
+ dimensionRawColumnChunks[chunkIndex].decodeColumnPage(pageId);
+ }
+ }
+ }
+
+
+ MeasureRawColumnChunk[] measureRawColumnChunks =
+ new MeasureRawColumnChunk[blockExecutionInfo.getTotalNumberOfMeasureToRead()];
+ int numMeasureChunks = measureRawColumnChunks.length;
+
+ // read the measure chunk blocks which is not present
+ for (int chunkIndex = 0; chunkIndex < numMeasureChunks; chunkIndex++) {
+ if (null != rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ measureRawColumnChunks[chunkIndex] =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
+ }
+ }
+
+ int[][] allSelectedMeasureColumnIndexRange =
+ blockExecutionInfo.getAllSelectedMeasureIndexRange();
+ MeasureRawColumnChunk[] projectionListMeasureChunk = rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunks(fileReader, allSelectedMeasureColumnIndexRange);
+ for (int[] columnIndexRange : allSelectedMeasureColumnIndexRange) {
+ System.arraycopy(projectionListMeasureChunk, columnIndexRange[0], measureRawColumnChunks,
+ columnIndexRange[0], columnIndexRange[1] + 1 - columnIndexRange[0]);
+ }
+ /*
+ * in case projection if the projected measure are not loaded in the ColumnPage
+ * then loading them
+ */
+ int[] projectionListMeasureIndexes = blockExecutionInfo.getProjectionListMeasureIndexes();
+ for (int projectionListMeasureIndex : projectionListMeasureIndexes) {
+ if (null == measureRawColumnChunks[projectionListMeasureIndex]) {
+ measureRawColumnChunks[projectionListMeasureIndex] = rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(fileReader, projectionListMeasureIndex);
+ }
+ }
+ ColumnPage[][] measureColumnPages = new ColumnPage[numMeasureChunks][numPages];
+ for (int chunkIndex = 0; chunkIndex < numMeasureChunks; chunkIndex++) {
+ if (measureRawColumnChunks[chunkIndex] != null) {
+ for (int pageId = 0; pageId < numPages; pageId++) {
+ measureColumnPages[chunkIndex][pageId] =
+ measureRawColumnChunks[chunkIndex].decodeColumnPage(pageId);
+ }
+ }
+ }
+
+ scannedResult.setDimensionColumnPages(dimensionColumnPages);
+ scannedResult.setPageFilteredRowId(pageFilteredRowId);
+ scannedResult.setMeasureColumnPages(measureColumnPages);
+ scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
+ scannedResult.setPageFilteredRowCount(pageFilteredRowCount);
+ // adding statistics for carbon scan time
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime - dimensionReadTime));
+ QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+ readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+ readTime.getCount() + dimensionReadTime);
+ return scannedResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
new file mode 100644
index 0000000..f0211dc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.scanner.impl;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
+import org.apache.carbondata.core.scan.result.impl.NonFilterQueryScannedResult;
+import org.apache.carbondata.core.scan.scanner.BlockletScanner;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+
+/**
+ * Blocklet scanner to do full scan of a blocklet,
+ * returning all projection and filter column chunks
+ */
+public class BlockletFullScanner implements BlockletScanner {
+
+ /**
+ * block execution info
+ */
+ protected BlockExecutionInfo blockExecutionInfo;
+
+ private QueryStatisticsModel queryStatisticsModel;
+
+ private BlockletScannedResult emptyResult;
+
+ public BlockletFullScanner(BlockExecutionInfo tableBlockExecutionInfos,
+ QueryStatisticsModel queryStatisticsModel) {
+ this.blockExecutionInfo = tableBlockExecutionInfos;
+ this.queryStatisticsModel = queryStatisticsModel;
+ }
+
+ @Override
+ public BlockletScannedResult scanBlocklet(
+ RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws IOException, FilterUnsupportedException {
+ long startTime = System.currentTimeMillis();
+ BlockletScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
+ QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
+ totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
+ totalBlockletStatistic.getCount() + 1);
+ QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
+ validScannedBlockletStatistic
+ .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
+ validScannedBlockletStatistic.getCount() + 1);
+ // adding statistics for valid number of pages
+ QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
+ validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
+ validPages.getCount() + rawBlockletColumnChunks.getDataBlock().numberOfPages());
+ // adding statistics for number of pages
+ QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
+ totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
+ totalPagesScanned.getCount() + rawBlockletColumnChunks.getDataBlock().numberOfPages());
+ scannedResult.setBlockletId(
+ blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR +
+ rawBlockletColumnChunks.getDataBlock().blockletIndex());
+ if (!blockExecutionInfo.isPrefetchBlocklet()) {
+ readBlocklet(rawBlockletColumnChunks);
+ }
+ DimensionRawColumnChunk[] dimensionRawColumnChunks =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks();
+ DimensionColumnPage[][] dimensionColumnDataChunks =
+ new DimensionColumnPage[dimensionRawColumnChunks.length][rawBlockletColumnChunks
+ .getDataBlock().numberOfPages()];
+ MeasureRawColumnChunk[] measureRawColumnChunks =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks();
+ ColumnPage[][] measureColumnPages =
+ new ColumnPage[measureRawColumnChunks.length][rawBlockletColumnChunks.getDataBlock()
+ .numberOfPages()];
+ scannedResult.setDimensionColumnPages(dimensionColumnDataChunks);
+ scannedResult.setMeasureColumnPages(measureColumnPages);
+ scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
+ if (blockExecutionInfo.isPrefetchBlocklet()) {
+ for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
+ if (dimensionRawColumnChunks[i] != null) {
+ dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].decodeAllColumnPages();
+ }
+ }
+ for (int i = 0; i < measureRawColumnChunks.length; i++) {
+ if (measureRawColumnChunks[i] != null) {
+ measureColumnPages[i] = measureRawColumnChunks[i].decodeAllColumnPages();
+ }
+ }
+ }
+ int[] numberOfRows = null;
+ if (blockExecutionInfo.getAllSelectedDimensionColumnIndexRange().length > 0) {
+ for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
+ if (dimensionRawColumnChunks[i] != null) {
+ numberOfRows = dimensionRawColumnChunks[i].getRowCount();
+ break;
+ }
+ }
+ } else if (blockExecutionInfo.getAllSelectedMeasureIndexRange().length > 0) {
+ for (int i = 0; i < measureRawColumnChunks.length; i++) {
+ if (measureRawColumnChunks[i] != null) {
+ numberOfRows = measureRawColumnChunks[i].getRowCount();
+ break;
+ }
+ }
+ }
+
+ // count(*) case there would not be any dimensions are measures selected.
+ if (numberOfRows == null) {
+ numberOfRows = new int[rawBlockletColumnChunks.getDataBlock().numberOfPages()];
+ for (int i = 0; i < numberOfRows.length; i++) {
+ numberOfRows[i] =
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ }
+ int lastPageSize = rawBlockletColumnChunks.getDataBlock().numRows()
+ % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ ;
+ if (lastPageSize > 0) {
+ numberOfRows[numberOfRows.length - 1] = lastPageSize;
+ }
+ }
+ scannedResult.setPageFilteredRowCount(numberOfRows);
+ if (!blockExecutionInfo.isPrefetchBlocklet()) {
+ scannedResult.fillDataChunks();
+ }
+ // adding statistics for carbon scan time
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime));
+ return scannedResult;
+ }
+
+ @Override
+ public void readBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws IOException {
+ long startTime = System.currentTimeMillis();
+ DimensionRawColumnChunk[] dimensionRawColumnChunks = rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunks(rawBlockletColumnChunks.getFileReader(),
+ blockExecutionInfo.getAllSelectedDimensionColumnIndexRange());
+ rawBlockletColumnChunks.setDimensionRawColumnChunks(dimensionRawColumnChunks);
+ MeasureRawColumnChunk[] measureRawColumnChunks = rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunks(rawBlockletColumnChunks.getFileReader(),
+ blockExecutionInfo.getAllSelectedMeasureIndexRange());
+ rawBlockletColumnChunks.setMeasureRawColumnChunks(measureRawColumnChunks);
+ // adding statistics for carbon read time
+ QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+ readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+ readTime.getCount() + (System.currentTimeMillis() - startTime));
+ }
+
+ BlockletScannedResult createEmptyResult() {
+ if (emptyResult == null) {
+ emptyResult = new NonFilterQueryScannedResult(blockExecutionInfo);
+ emptyResult.setPageFilteredRowCount(new int[0]);
+ emptyResult.setPageFilteredRowId(new int[0][]);
+ }
+ return emptyResult;
+ }
+
+ @Override public boolean isScanRequired(DataRefNode dataBlock) {
+ // For non filter it is always true
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
deleted file mode 100644
index e77093b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.scanner.impl;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
-import org.apache.carbondata.core.scan.result.impl.FilterQueryScannedResult;
-import org.apache.carbondata.core.scan.scanner.AbstractBlockletScanner;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.util.BitSetGroup;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-/**
- * Below class will be used for filter query processing
- * this class will be first apply the filter then it will read the block if
- * required and return the scanned result
- */
-public class FilterScanner extends AbstractBlockletScanner {
-
- /**
- * filter tree
- */
- private FilterExecuter filterExecuter;
- /**
- * this will be used to apply min max
- * this will be useful for dimension column which is on the right side
- * as node finder will always give tentative blocks, if column data stored individually
- * and data is in sorted order then we can check whether filter is in the range of min max or not
- * if it present then only we can apply filter on complete data.
- * this will be very useful in case of sparse data when rows are
- * repeating.
- */
- private boolean isMinMaxEnabled;
-
- private QueryStatisticsModel queryStatisticsModel;
-
- private boolean useBitSetPipeLine;
-
- public FilterScanner(BlockExecutionInfo blockExecutionInfo,
- QueryStatisticsModel queryStatisticsModel) {
- super(blockExecutionInfo);
- // to check whether min max is enabled or not
- String minMaxEnableValue = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_QUERY_MIN_MAX_ENABLED,
- CarbonCommonConstants.MIN_MAX_DEFAULT_VALUE);
- if (null != minMaxEnableValue) {
- isMinMaxEnabled = Boolean.parseBoolean(minMaxEnableValue);
- }
- // get the filter tree
- this.filterExecuter = blockExecutionInfo.getFilterExecuterTree();
- this.queryStatisticsModel = queryStatisticsModel;
-
- String useBitSetPipeLine = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.BITSET_PIPE_LINE,
- CarbonCommonConstants.BITSET_PIPE_LINE_DEFAULT);
- if (null != useBitSetPipeLine) {
- this.useBitSetPipeLine = Boolean.parseBoolean(useBitSetPipeLine);
- }
- }
-
- /**
- * Below method will be used to process the block
- *
- * @param blocksChunkHolder block chunk holder which holds the data
- * @throws FilterUnsupportedException
- */
- @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws IOException, FilterUnsupportedException {
- return fillScannedResult(blocksChunkHolder);
- }
-
- @Override public boolean isScanRequired(BlocksChunkHolder blocksChunkHolder) throws IOException {
- // adding statistics for number of pages
- QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
- totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
- totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
- // apply min max
- if (isMinMaxEnabled) {
- BitSet bitSet = null;
- // check for implicit include filter instance
- if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
- String blockletId = blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR
- + blocksChunkHolder.getDataBlock().blockletId();
- bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
- .isFilterValuesPresentInBlockOrBlocklet(
- blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
- blocksChunkHolder.getDataBlock().getColumnsMinValue(), blockletId);
- } else {
- bitSet = this.filterExecuter
- .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
- blocksChunkHolder.getDataBlock().getColumnsMinValue());
- }
- if (bitSet.isEmpty()) {
- CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(),
- blocksChunkHolder.getMeasureRawDataChunk());
- return false;
- }
- }
- return true;
- }
-
- @Override public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
- long startTime = System.currentTimeMillis();
- this.filterExecuter.readBlocks(blocksChunkHolder);
- // adding statistics for carbon read time
- QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
- readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
- readTime.getCount() + (System.currentTimeMillis() - startTime));
- }
-
- /**
- * This method will process the data in below order
- * 1. first apply min max on the filter tree and check whether any of the filter
- * is fall on the range of min max, if not then return empty result
- * 2. If filter falls on min max range then apply filter on actual
- * data and get the filtered row index
- * 3. if row index is empty then return the empty result
- * 4. if row indexes is not empty then read only those blocks(measure or dimension)
- * which was present in the query but not present in the filter, as while applying filter
- * some of the blocks where already read and present in chunk holder so not need to
- * read those blocks again, this is to avoid reading of same blocks which was already read
- * 5. Set the blocks and filter indexes to result
- *
- * @param blocksChunkHolder
- * @throws FilterUnsupportedException
- */
- private AbstractScannedResult fillScannedResult(BlocksChunkHolder blocksChunkHolder)
- throws FilterUnsupportedException, IOException {
- long startTime = System.currentTimeMillis();
- QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
- totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
- totalBlockletStatistic.getCount() + 1);
- // apply filter on actual data
- BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder, useBitSetPipeLine);
- // if indexes is empty then return with empty result
- if (bitSetGroup.isEmpty()) {
- CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(),
- blocksChunkHolder.getMeasureRawDataChunk());
-
- QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
- scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
- scanTime.getCount() + (System.currentTimeMillis() - startTime));
-
- QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.PAGE_SCANNED);
- scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
- scannedPages.getCount() + bitSetGroup.getScannedPages());
- return createEmptyResult();
- }
-
- AbstractScannedResult scannedResult = new FilterQueryScannedResult(blockExecutionInfo);
- scannedResult.setBlockletId(
- blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
- .getDataBlock().blockletId());
- // valid scanned blocklet
- QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
- validScannedBlockletStatistic
- .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
- validScannedBlockletStatistic.getCount() + 1);
- // adding statistics for valid number of pages
- QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
- validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
- validPages.getCount() + bitSetGroup.getValidPages());
- QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.PAGE_SCANNED);
- scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
- scannedPages.getCount() + bitSetGroup.getScannedPages());
- int[] rowCount = new int[bitSetGroup.getNumberOfPages()];
- // get the row indexes from bot set
- int[][] indexesGroup = new int[bitSetGroup.getNumberOfPages()][];
- for (int k = 0; k < indexesGroup.length; k++) {
- BitSet bitSet = bitSetGroup.getBitSet(k);
- if (bitSet != null && !bitSet.isEmpty()) {
- int[] indexes = new int[bitSet.cardinality()];
- int index = 0;
- for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
- indexes[index++] = i;
- }
- rowCount[k] = indexes.length;
- indexesGroup[k] = indexes;
- }
- }
- FileHolder fileReader = blocksChunkHolder.getFileReader();
- int[][] allSelectedDimensionBlocksIndexes =
- blockExecutionInfo.getAllSelectedDimensionBlocksIndexes();
- long dimensionReadTime = System.currentTimeMillis();
- DimensionRawColumnChunk[] projectionListDimensionChunk = blocksChunkHolder.getDataBlock()
- .getDimensionChunks(fileReader, allSelectedDimensionBlocksIndexes);
- dimensionReadTime = System.currentTimeMillis() - dimensionReadTime;
-
- DimensionRawColumnChunk[] dimensionRawColumnChunks =
- new DimensionRawColumnChunk[blockExecutionInfo.getTotalNumberDimensionBlock()];
- // read dimension chunk blocks from file which is not present
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (null != blocksChunkHolder.getDimensionRawDataChunk()[i]) {
- dimensionRawColumnChunks[i] = blocksChunkHolder.getDimensionRawDataChunk()[i];
- }
- }
- for (int i = 0; i < allSelectedDimensionBlocksIndexes.length; i++) {
- for (int j = allSelectedDimensionBlocksIndexes[i][0];
- j <= allSelectedDimensionBlocksIndexes[i][1]; j++) {
- dimensionRawColumnChunks[j] = projectionListDimensionChunk[j];
- }
- }
- long dimensionReadTime1 = System.currentTimeMillis();
- /**
- * in case projection if the projected dimension are not loaded in the dimensionColumnDataChunk
- * then loading them
- */
- int[] projectionListDimensionIndexes = blockExecutionInfo.getProjectionListDimensionIndexes();
- int projectionListDimensionIndexesLength = projectionListDimensionIndexes.length;
- for (int i = 0; i < projectionListDimensionIndexesLength; i++) {
- if (null == dimensionRawColumnChunks[projectionListDimensionIndexes[i]]) {
- dimensionRawColumnChunks[projectionListDimensionIndexes[i]] =
- blocksChunkHolder.getDataBlock()
- .getDimensionChunk(fileReader, projectionListDimensionIndexes[i]);
- }
- }
- dimensionReadTime += (System.currentTimeMillis() - dimensionReadTime1);
- dimensionReadTime1 = System.currentTimeMillis();
- MeasureRawColumnChunk[] measureRawColumnChunks =
- new MeasureRawColumnChunk[blockExecutionInfo.getTotalNumberOfMeasureBlock()];
- int[][] allSelectedMeasureBlocksIndexes =
- blockExecutionInfo.getAllSelectedMeasureBlocksIndexes();
- MeasureRawColumnChunk[] projectionListMeasureChunk = blocksChunkHolder.getDataBlock()
- .getMeasureChunks(fileReader, allSelectedMeasureBlocksIndexes);
- dimensionReadTime += System.currentTimeMillis() - dimensionReadTime1;
- // read the measure chunk blocks which is not present
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (null != blocksChunkHolder.getMeasureRawDataChunk()[i]) {
- measureRawColumnChunks[i] = blocksChunkHolder.getMeasureRawDataChunk()[i];
- }
- }
- for (int i = 0; i < allSelectedMeasureBlocksIndexes.length; i++) {
- for (int j = allSelectedMeasureBlocksIndexes[i][0];
- j <= allSelectedMeasureBlocksIndexes[i][1]; j++) {
- measureRawColumnChunks[j] = projectionListMeasureChunk[j];
- }
- }
- dimensionReadTime1 = System.currentTimeMillis();
- /**
- * in case projection if the projected measure are not loaded in the ColumnPage
- * then loading them
- */
- int[] projectionListMeasureIndexes = blockExecutionInfo.getProjectionListMeasureIndexes();
- int projectionListMeasureIndexesLength = projectionListMeasureIndexes.length;
- for (int i = 0; i < projectionListMeasureIndexesLength; i++) {
- if (null == measureRawColumnChunks[projectionListMeasureIndexes[i]]) {
- measureRawColumnChunks[projectionListMeasureIndexes[i]] = blocksChunkHolder.getDataBlock()
- .getMeasureChunk(fileReader, projectionListMeasureIndexes[i]);
- }
- }
- dimensionReadTime += System.currentTimeMillis() - dimensionReadTime1;
- DimensionColumnDataChunk[][] dimensionColumnDataChunks =
- new DimensionColumnDataChunk[dimensionRawColumnChunks.length][indexesGroup.length];
- ColumnPage[][] columnPages =
- new ColumnPage[measureRawColumnChunks.length][indexesGroup.length];
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (dimensionRawColumnChunks[i] != null) {
- for (int j = 0; j < indexesGroup.length; j++) {
- dimensionColumnDataChunks[i][j] = dimensionRawColumnChunks[i].convertToDimColDataChunk(j);
- }
- }
- }
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (measureRawColumnChunks[i] != null) {
- for (int j = 0; j < indexesGroup.length; j++) {
- columnPages[i][j] = measureRawColumnChunks[i].convertToColumnPage(j);
- }
- }
- }
- scannedResult.setDimensionChunks(dimensionColumnDataChunks);
- scannedResult.setIndexes(indexesGroup);
- scannedResult.setMeasureChunks(columnPages);
- scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
- scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
- scannedResult.setNumberOfRows(rowCount);
- // adding statistics for carbon scan time
- QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
- scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
- scanTime.getCount() + (System.currentTimeMillis() - startTime - dimensionReadTime));
- QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
- readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
- readTime.getCount() + dimensionReadTime);
- return scannedResult;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
deleted file mode 100644
index 1373ed5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.scanner.impl;
-
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.scanner.AbstractBlockletScanner;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-
-/**
- * Non filter processor which will be used for non filter query
- * In case of non filter query we just need to read all the blocks requested in the
- * query and pass it to scanned result
- */
-public class NonFilterScanner extends AbstractBlockletScanner {
-
- public NonFilterScanner(BlockExecutionInfo blockExecutionInfo,
- QueryStatisticsModel queryStatisticsModel) {
- super(blockExecutionInfo);
- super.queryStatisticsModel = queryStatisticsModel;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
index 2f981b5..6faae03 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
@@ -30,22 +30,17 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper>, Serializa
* to store key which is generated using
* key generator
*/
- protected byte[] dictionaryKey;
+ private byte[] dictionaryKey;
/**
* to store no dictionary column data
*/
- protected byte[][] complexTypesKeys;
+ private byte[][] complexTypesKeys;
/**
* to store no dictionary column data
*/
- protected byte[][] noDictionaryKeys;
-
- /**
- * contains value of implicit columns in byte array format
- */
- protected byte[] implicitColumnByteArray;
+ private byte[][] noDictionaryKeys;
public ByteArrayWrapper() {
}
@@ -91,16 +86,6 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper>, Serializa
}
/**
- * to get the no dictionary column data
- *
- * @param index of the no dictionary key
- * @return no dictionary key for the index
- */
- public byte[] getComplexTypeByIndex(int index) {
- return this.complexTypesKeys[index];
- }
-
- /**
* to generate the hash code
*/
@Override public int hashCode() {
@@ -201,30 +186,10 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper>, Serializa
}
/**
- * @return the complexTypesKeys
- */
- public byte[][] getComplexTypesKeys() {
- return complexTypesKeys;
- }
-
- /**
* @param complexTypesKeys the complexTypesKeys to set
*/
public void setComplexTypesKeys(byte[][] complexTypesKeys) {
this.complexTypesKeys = complexTypesKeys;
}
- /**
- * @return
- */
- public byte[] getImplicitColumnByteArray() {
- return implicitColumnByteArray;
- }
-
- /**
- * @param implicitColumnByteArray
- */
- public void setImplicitColumnByteArray(byte[] implicitColumnByteArray) {
- this.implicitColumnByteArray = implicitColumnByteArray;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java b/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
deleted file mode 100644
index 8a37d01..0000000
--- a/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.stats;
-
-import java.io.Serializable;
-
-public class PartitionStatistic implements Serializable {
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
index 55f0882..ed60d37 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
@@ -37,7 +37,4 @@ public class QueryStatisticsModel {
return statisticsTypeAndObjMap;
}
- public void setStatisticsTypeAndObjMap(Map<String, QueryStatistic> statisticsTypeAndObjMap) {
- this.statisticsTypeAndObjMap = statisticsTypeAndObjMap;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index 73a665d..d0c8e93 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -341,15 +341,6 @@ public class LoadMetadataDetails implements Serializable {
}
/**
- * To get isDeleted property.
- *
- * @return isDeleted
- */
- public String getIsDeleted() {
- return isDeleted;
- }
-
- /**
* To set isDeleted property.
*
* @param isDeleted
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index 6af0304..e02f246 100755
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -652,22 +652,6 @@ public class SegmentStatusManager {
return "";
}
- /**
- * getting the task numbers present in the segment.
- * @param segmentId
- * @return
- */
- public List<String> getUpdatedTasksDetailsForSegment(String segmentId, SegmentUpdateStatusManager
- updateStatusManager) {
- List<String> taskList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- List<String> list = updateStatusManager.getUpdateDeltaFiles(segmentId);
- for (String eachFileName : list) {
- taskList.add(CarbonTablePath.DataFileUtil.getTaskNo(eachFileName));
- }
- return taskList;
- }
-
-
public static class ValidAndInvalidSegmentsInfo {
private final List<String> listOfValidSegments;
private final List<String> listOfValidUpdatedSegments;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index d4ef5c6..66f7a12 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -47,7 +47,6 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
import org.apache.carbondata.core.mutate.TupleIdEnum;
import org.apache.carbondata.core.mutate.UpdateVO;
-import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -140,14 +139,6 @@ public class SegmentUpdateStatusManager {
}
/**
- *
- * @param loadMetadataDetails
- */
- public void setLoadMetadataDetails(LoadMetadataDetails[] loadMetadataDetails) {
- this.segmentDetails = loadMetadataDetails;
- }
-
- /**
* Returns the UpdateStatus Details.
* @return
*/
@@ -174,18 +165,6 @@ public class SegmentUpdateStatusManager {
}
/**
- * Returns all delete delta files of specified block
- *
- * @param tupleId
- * @return
- * @throws Exception
- */
- public List<String> getDeleteDeltaFiles(String tupleId) throws Exception {
- return getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT);
- }
-
-
- /**
* Returns all update delta files of specified Segment.
*
* @param segmentId
@@ -248,20 +227,6 @@ public class SegmentUpdateStatusManager {
}
/**
- * Returns all deleted records of specified block
- *
- * @param tupleId
- * @return
- * @throws Exception
- */
- public Map<Integer, Integer[]> getDeleteDeltaDataFromAllFiles(String tupleId) throws Exception {
- List<String> deltaFiles = getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT);
- CarbonDeleteFilesDataReader dataReader = new CarbonDeleteFilesDataReader();
- String blockletId = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCKLET_ID);
- return dataReader.getDeleteDataFromAllFiles(deltaFiles, blockletId);
- }
-
- /**
* Below method will be used to get all the delete delta files based on block name
*
* @param blockFilePath actual block filePath
@@ -775,41 +740,6 @@ public class SegmentUpdateStatusManager {
}
/**
- * compares passed time stamp with status file delete timestamp and
- * returns latest timestamp from status file if both are not equal
- * returns null otherwise
- *
- * @param completeBlockName
- * @param timestamp
- * @return
- */
- public String getTimestampForRefreshCache(String completeBlockName, String timestamp) {
- long cacheTimestamp = 0;
- if (null != timestamp) {
- cacheTimestamp = CarbonUpdateUtil.getTimeStampAsLong(timestamp);
- }
- String blockName = CarbonTablePath.addDataPartPrefix(CarbonUpdateUtil.getBlockName(
- CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.BLOCK_ID)));
- String segmentId =
- CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.SEGMENT_ID);
- SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray =
- readLoadMetadata();
- for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) {
- if (segmentId.equalsIgnoreCase(block.getSegmentName()) &&
- block.getBlockName().equalsIgnoreCase(blockName) &&
- !CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) {
- long deleteTimestampFromStatusFile = block.getDeleteDeltaEndTimeAsLong();
- if (Long.compare(deleteTimestampFromStatusFile, cacheTimestamp) == 0) {
- return null;
- } else {
- return block.getDeleteDeltaEndTimestamp();
- }
- }
- }
- return null;
- }
-
- /**
* This method closes the streams
*
* @param streams - streams to close.
@@ -828,85 +758,7 @@ public class SegmentUpdateStatusManager {
}
}
}
- /**
- * Get the invalid tasks in that segment.
- * @param segmentId
- * @return
- */
- public List<String> getInvalidBlockList(String segmentId) {
-
- // get the original fact file timestamp from the table status file.
- List<String> listOfInvalidBlocks = new ArrayList<String>();
- SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
- LoadMetadataDetails[] segmentDetails =
- ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
- long timestampOfOriginalFacts = 0;
-
- String startTimestampOfUpdate = "" ;
- String endTimestampOfUpdate = "";
-
- for (LoadMetadataDetails segment : segmentDetails) {
- // find matching segment and return timestamp.
- if (segment.getLoadName().equalsIgnoreCase(segmentId)) {
- timestampOfOriginalFacts = segment.getLoadStartTime();
- startTimestampOfUpdate = segment.getUpdateDeltaStartTimestamp();
- endTimestampOfUpdate = segment.getUpdateDeltaEndTimestamp();
- }
- }
-
- if (startTimestampOfUpdate.isEmpty()) {
- return listOfInvalidBlocks;
-
- }
-
- // now after getting the original fact timestamp, what ever is remaining
- // files need to cross check it with table status file.
-
- // filter out the fact files.
-
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
- CarbonFile segDir =
- FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
-
- final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimestampOfUpdate);
- final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimestampOfUpdate);
- final Long timeStampOriginalFactFinal =
- timestampOfOriginalFacts;
-
- CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {
-
- @Override public boolean accept(CarbonFile pathName) {
- String fileName = pathName.getName();
- if (fileName.endsWith(CarbonCommonConstants.UPDATE_DELTA_FILE_EXT)) {
- String firstPart = fileName.substring(0, fileName.indexOf('.'));
-
- long timestamp = Long.parseLong(firstPart
- .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
- firstPart.length()));
- if (Long.compare(timestamp, endTimeStampFinal) <= 0
- && Long.compare(timestamp, startTimeStampFinal) >= 0) {
- return false;
- }
- if (Long.compare(timestamp, timeStampOriginalFactFinal) == 0) {
- return false;
- }
- // take the rest of files as they are invalid.
- return true;
- }
- return false;
- }
- });
- // gather the task numbers.
- for (CarbonFile updateFiles : files) {
- listOfInvalidBlocks.add(updateFiles.getName());
- }
-
- return listOfInvalidBlocks;
- }
/**
* Returns the invalid timestamp range of a segment.
* @param segmentId
@@ -932,12 +784,11 @@ public class SegmentUpdateStatusManager {
}
/**
*
- * @param segmentId
* @param block
* @param needCompleteList
* @return
*/
- public CarbonFile[] getDeleteDeltaInvalidFilesList(final String segmentId,
+ public CarbonFile[] getDeleteDeltaInvalidFilesList(
final SegmentUpdateDetails block, final boolean needCompleteList,
CarbonFile[] allSegmentFiles, boolean isAbortedFile) {
@@ -983,12 +834,11 @@ public class SegmentUpdateStatusManager {
/**
*
- * @param blockName
* @param allSegmentFiles
* @return
*/
- public CarbonFile[] getAllBlockRelatedFiles(String blockName, CarbonFile[] allSegmentFiles,
- String actualBlockName) {
+ public CarbonFile[] getAllBlockRelatedFiles(CarbonFile[] allSegmentFiles,
+ String actualBlockName) {
List<CarbonFile> files = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
for (CarbonFile eachFile : allSegmentFiles) {
[49/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap]
Datamap FineGrain implementation
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
index 78544d3..fe0bbcf 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.datamap.examples;
import java.io.BufferedWriter;
import java.io.DataOutputStream;
-import java.io.File;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
@@ -29,17 +28,18 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import com.google.gson.Gson;
-public class MinMaxDataWriter implements DataMapWriter {
+public class MinMaxDataWriter extends AbstractDataMapWriter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(TableInfo.class.getName());
@@ -50,17 +50,23 @@ public class MinMaxDataWriter implements DataMapWriter {
private Map<Integer, BlockletMinMax> blockMinMaxMap;
- private String blockPath;
+ private String dataWritePath;
+ public MinMaxDataWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String dataWritePath) {
+ super(identifier, segmentId, dataWritePath);
+ this.identifier = identifier;
+ this.segmentId = segmentId;
+ this.dataWritePath = dataWritePath;
+ }
- @Override public void onBlockStart(String blockId, String blockPath) {
+ @Override public void onBlockStart(String blockId) {
pageLevelMax = null;
pageLevelMin = null;
blockletLevelMax = null;
blockletLevelMin = null;
blockMinMaxMap = null;
blockMinMaxMap = new HashMap<Integer, BlockletMinMax>();
- this.blockPath = blockPath;
}
@Override public void onBlockEnd(String blockId) {
@@ -161,7 +167,7 @@ public class MinMaxDataWriter implements DataMapWriter {
List<MinMaxIndexBlockDetails> tempMinMaxIndexBlockDetails = null;
tempMinMaxIndexBlockDetails = loadBlockDetails();
try {
- writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockPath, blockId);
+ writeMinMaxIndexFile(tempMinMaxIndexBlockDetails, blockId);
} catch (IOException ex) {
LOGGER.info(" Unable to write the file");
}
@@ -178,7 +184,6 @@ public class MinMaxDataWriter implements DataMapWriter {
tmpminMaxIndexBlockDetails.setMinValues(blockMinMaxMap.get(index).getMin());
tmpminMaxIndexBlockDetails.setMaxValues(blockMinMaxMap.get(index).getMax());
tmpminMaxIndexBlockDetails.setBlockletId(index);
- tmpminMaxIndexBlockDetails.setFilePath(this.blockPath);
minMaxIndexBlockDetails.add(tmpminMaxIndexBlockDetails);
}
return minMaxIndexBlockDetails;
@@ -187,22 +192,19 @@ public class MinMaxDataWriter implements DataMapWriter {
/**
* Write the data to a file. This is JSON format file.
* @param minMaxIndexBlockDetails
- * @param blockPath
* @param blockId
* @throws IOException
*/
public void writeMinMaxIndexFile(List<MinMaxIndexBlockDetails> minMaxIndexBlockDetails,
- String blockPath, String blockId) throws IOException {
- String filePath = blockPath.substring(0, blockPath.lastIndexOf(File.separator) + 1) + blockId
- + ".minmaxindex";
+ String blockId) throws IOException {
+ String filePath = dataWritePath +"/" + blockId + ".minmaxindex";
BufferedWriter brWriter = null;
DataOutputStream dataOutStream = null;
try {
FileFactory.createNewFile(filePath, FileFactory.getFileType(filePath));
dataOutStream = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath));
Gson gsonObjectToWrite = new Gson();
- brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream,
- CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT));
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream, "UTF-8"));
String minmaxIndexData = gsonObjectToWrite.toJson(minMaxIndexBlockDetails);
brWriter.write(minmaxIndexData);
} catch (IOException ioe) {
@@ -215,7 +217,11 @@ public class MinMaxDataWriter implements DataMapWriter {
dataOutStream.flush();
}
CarbonUtil.closeStreams(brWriter, dataOutStream);
+ commitFile(filePath);
}
}
+ @Override public void finish() throws IOException {
+
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
index 0596db5..93a453e 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexBlockDetails.java
@@ -33,11 +33,6 @@ public class MinMaxIndexBlockDetails implements Serializable {
private byte[][] maxValues;
/**
- * filePath pointing to the block.
- */
- private String filePath;
-
- /**
* BlockletID of the block.
*/
private Integer BlockletId;
@@ -59,14 +54,6 @@ public class MinMaxIndexBlockDetails implements Serializable {
this.maxValues = maxValues;
}
- public String getFilePath() {
- return filePath;
- }
-
- public void setFilePath(String filePath) {
- this.filePath = filePath;
- }
-
public Integer getBlockletId() {
return BlockletId;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index c1ef14d..7d91806 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -86,6 +86,8 @@ public class CarbonInputSplit extends FileSplit
private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
+ private String dataMapWritePath;
+
public CarbonInputSplit() {
segmentId = null;
taskId = "0";
@@ -97,7 +99,8 @@ public class CarbonInputSplit extends FileSplit
}
private CarbonInputSplit(String segmentId, String blockletId, Path path, long start, long length,
- String[] locations, ColumnarFormatVersion version, String[] deleteDeltaFiles) {
+ String[] locations, ColumnarFormatVersion version, String[] deleteDeltaFiles,
+ String dataMapWritePath) {
super(path, start, length, locations);
this.segmentId = segmentId;
String taskNo = CarbonTablePath.DataFileUtil.getTaskNo(path.getName());
@@ -110,12 +113,13 @@ public class CarbonInputSplit extends FileSplit
this.invalidSegments = new ArrayList<>();
this.version = version;
this.deleteDeltaFiles = deleteDeltaFiles;
+ this.dataMapWritePath = dataMapWritePath;
}
public CarbonInputSplit(String segmentId, String blockletId, Path path, long start, long length,
String[] locations, int numberOfBlocklets, ColumnarFormatVersion version,
String[] deleteDeltaFiles) {
- this(segmentId, blockletId, path, start, length, locations, version, deleteDeltaFiles);
+ this(segmentId, blockletId, path, start, length, locations, version, deleteDeltaFiles, null);
this.numberOfBlocklets = numberOfBlocklets;
}
@@ -165,9 +169,9 @@ public class CarbonInputSplit extends FileSplit
}
public static CarbonInputSplit from(String segmentId, String blockletId, FileSplit split,
- ColumnarFormatVersion version) throws IOException {
+ ColumnarFormatVersion version, String dataMapWritePath) throws IOException {
return new CarbonInputSplit(segmentId, blockletId, split.getPath(), split.getStart(),
- split.getLength(), split.getLocations(), version, null);
+ split.getLength(), split.getLocations(), version, null, dataMapWritePath);
}
public static List<TableBlockInfo> createBlocks(List<CarbonInputSplit> splitList) {
@@ -181,6 +185,7 @@ public class CarbonInputSplit extends FileSplit
split.getSegmentId(), split.getLocations(), split.getLength(), blockletInfos,
split.getVersion(), split.getDeleteDeltaFiles());
blockInfo.setDetailInfo(split.getDetailInfo());
+ blockInfo.setDataMapWriterPath(split.dataMapWritePath);
tableBlockInfoList.add(blockInfo);
} catch (IOException e) {
throw new RuntimeException("fail to get location of split: " + split, e);
@@ -230,6 +235,10 @@ public class CarbonInputSplit extends FileSplit
detailInfo = new BlockletDetailInfo();
detailInfo.readFields(in);
}
+ boolean dataMapWriterPathExists = in.readBoolean();
+ if (dataMapWriterPathExists) {
+ dataMapWritePath = in.readUTF();
+ }
}
@Override public void write(DataOutput out) throws IOException {
@@ -252,6 +261,10 @@ public class CarbonInputSplit extends FileSplit
if (detailInfo != null) {
detailInfo.write(out);
}
+ out.writeBoolean(dataMapWritePath != null);
+ if (dataMapWritePath != null) {
+ out.writeUTF(dataMapWritePath);
+ }
}
public List<String> getInvalidSegments() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 24f5713..ad5a1c9 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -34,6 +34,7 @@ import java.util.Set;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
+import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -716,16 +717,17 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// get tokens for all the required FileSystem for table path
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
new Path[] { new Path(absoluteTableIdentifier.getTablePath()) }, job.getConfiguration());
-
- TableDataMap blockletMap = DataMapStoreManager.getInstance()
- .getDataMap(absoluteTableIdentifier, BlockletDataMap.NAME,
- BlockletDataMapFactory.class.getName());
+ boolean distributedCG = Boolean.parseBoolean(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT));
+ TableDataMap blockletMap =
+ DataMapStoreManager.getInstance().chooseDataMap(absoluteTableIdentifier);
DataMapJob dataMapJob = getDataMapJob(job.getConfiguration());
List<String> partitionsToPrune = getPartitionsToPrune(job.getConfiguration());
List<ExtendedBlocklet> prunedBlocklets;
- if (dataMapJob != null) {
+ if (distributedCG || blockletMap.getDataMapFactory().getDataMapType() == DataMapType.FG) {
DistributableDataMapFormat datamapDstr =
- new DistributableDataMapFormat(absoluteTableIdentifier, BlockletDataMap.NAME,
+ new DistributableDataMapFormat(absoluteTableIdentifier, blockletMap.getDataMapName(),
segmentIds, partitionsToPrune,
BlockletDataMapFactory.class.getName());
prunedBlocklets = dataMapJob.execute(datamapDstr, resolver);
@@ -778,7 +780,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
org.apache.carbondata.hadoop.CarbonInputSplit.from(blocklet.getSegmentId(),
blocklet.getBlockletId(), new FileSplit(new Path(blocklet.getPath()), 0,
blocklet.getLength(), blocklet.getLocations()),
- ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()));
+ ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()),
+ blocklet.getDataMapWriterPath());
split.setDetailInfo(blocklet.getDetailInfo());
return split;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
new file mode 100644
index 0000000..4b6f231
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datastore.FileReader
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.Blocklet
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class CGDataMapFactory extends AbstractCoarseGrainDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapName: String = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier,
+ dataMapName: String): Unit = {
+ this.identifier = identifier
+ this.dataMapName = dataMapName
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter = {
+ new CGDataMapWriter(identifier, segmentId, dataWritePath, dataMapName)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segmentId: String): java.util.List[AbstractCoarseGrainDataMap] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map {f =>
+ val dataMap: AbstractCoarseGrainDataMap = new CGDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+
+ /**
+ * Get datamaps for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: AbstractCoarseGrainDataMap = new CGDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event): Unit = {
+ ???
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segmentId: String): java.util.List[DataMapDistributable] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d:DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segmentId: String): Unit = {
+
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(Seq("name").toList.asJava, new ArrayBuffer[ExpressionType]().toList.asJava)
+ }
+}
+
+class CGDataMap extends AbstractCoarseGrainDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))] = _
+ var FileReader: FileReader = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = FileReader.readInt(filePath, size-4)
+ val bytes = FileReader.readByteArray(filePath, size-footerLen-4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject().asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(
+ filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties,
+ partitions: java.util.List[String]): java.util.List[Blocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f=>
+ new Blocklet(f._1, f._2+"")
+ }.asJava
+ }
+
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) <= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) >= 0
+ }
+ tuples
+ }
+
+ private def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear() = {
+ ???
+ }
+
+ override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
+}
+
+class CGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segmentId: String,
+ dataWritePath: String,
+ dataMapName: String)
+ extends AbstractDataMapWriter(identifier, segmentId, dataWritePath) {
+
+ var currentBlockId: String = null
+ val cgwritepath = dataWritePath + "/" +
+ dataMapName + System.nanoTime() + ".datamap"
+ lazy val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(cgwritepath, FileFactory.getFileType(cgwritepath))
+ val blockletList = new ArrayBuffer[Array[Byte]]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]()
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ maxMin +=
+ ((currentBlockId+"", blockletId, (sorted.last, sorted.head)))
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[Array[Byte]]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += newBytes
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ blockletList += sorted.head
+ blockletList += sorted.last
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(cgwritepath)
+ }
+
+
+}
+
+class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test cg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ sql(
+ """
+ | CREATE TABLE datamap_test_cg(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test_cg")
+ // register datamap writer
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ table.getAbsoluteTableIdentifier,
+ classOf[CGDataMapFactory].getName, "cgdatamap")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_cg OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test_cg where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index 553e080..f694a6b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -20,25 +20,30 @@ package org.apache.carbondata.spark.testsuite.datamap
import java.util
import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.{DataFrame, SaveMode}
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.{DataFrame, SaveMode}
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.dev.{DataMap, DataMapFactory, DataMapWriter}
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.schema.FilterType
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.Event
-class C2DataMapFactory() extends DataMapFactory {
+class C2DataMapFactory() extends AbstractCoarseGrainDataMapFactory {
+
+ var identifier: AbsoluteTableIdentifier = _
override def init(identifier: AbsoluteTableIdentifier,
- dataMapName: String): Unit = {}
+ dataMapName: String): Unit = {
+ this.identifier = identifier
+ }
override def fireEvent(event: Event): Unit = ???
@@ -46,13 +51,14 @@ class C2DataMapFactory() extends DataMapFactory {
override def clear(): Unit = {}
- override def getDataMaps(distributable: DataMapDistributable): java.util.List[DataMap] = ???
+ override def getDataMaps(distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = ???
- override def getDataMaps(segmentId: String): util.List[DataMap] = ???
+ override def getDataMaps(segmentId: String): util.List[AbstractCoarseGrainDataMap] = ???
- override def createWriter(segmentId: String): DataMapWriter = DataMapWriterSuite.dataMapWriterC2Mock
+ override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter =
+ DataMapWriterSuite.dataMapWriterC2Mock(identifier, segmentId, dataWritePath)
- override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, FilterType.EQUALTO)
+ override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, List(ExpressionType.EQUALS).asJava)
/**
* Get all distributable objects of a segmentid
@@ -62,6 +68,7 @@ class C2DataMapFactory() extends DataMapFactory {
override def toDistributable(segmentId: String): util.List[DataMapDistributable] = {
???
}
+
}
class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
@@ -164,9 +171,12 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
}
object DataMapWriterSuite {
+
var callbackSeq: Seq[String] = Seq[String]()
- val dataMapWriterC2Mock = new DataMapWriter {
+ def dataMapWriterC2Mock(identifier: AbsoluteTableIdentifier, segmentId: String,
+ dataWritePath: String) =
+ new AbstractDataMapWriter(identifier, segmentId, dataWritePath) {
override def onPageAdded(
blockletId: Int,
@@ -191,9 +201,21 @@ object DataMapWriterSuite {
callbackSeq :+= s"blocklet start $blockletId"
}
- override def onBlockStart(blockId: String, blockPath: String): Unit = {
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String) = {
callbackSeq :+= s"block start $blockId"
}
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish() = {
+
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
new file mode 100644
index 0000000..d1bb65f
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -0,0 +1,440 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.dev.fgdatamap.{AbstractFineGrainDataMap, AbstractFineGrainDataMapFactory}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datastore.FileReader
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapName: String = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier,
+ dataMapName: String): Unit = {
+ this.identifier = identifier
+ this.dataMapName = dataMapName
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter = {
+ new FGDataMapWriter(identifier, segmentId, dataWritePath, dataMapName)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segmentId: String): java.util.List[AbstractFineGrainDataMap] = {
+ val file = FileFactory
+ .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val dataMap: AbstractFineGrainDataMap = new FGDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+ /**
+ * Get datamap for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[AbstractFineGrainDataMap]= {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: AbstractFineGrainDataMap = new FGDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segmentId: String): java.util.List[DataMapDistributable] = {
+ val file = FileFactory
+ .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d: DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event):Unit = {
+ ???
+ }
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segmentId: String): Unit = {
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(Seq("name").toList.asJava, new ArrayBuffer[ExpressionType]().toList.asJava)
+ }
+}
+
+class FGDataMap extends AbstractFineGrainDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)] = _
+ var FileReader: FileReader = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = FileReader.readInt(filePath, size - 4)
+ val bytes = FileReader.readByteArray(filePath, size - footerLen - 4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(
+ filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties,
+ partitions: java.util.List[String]): java.util.List[FineGrainBlocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f =>
+ readAndFindData(f, value(0).getBytes())
+ }.filter(_.isDefined).map(_.get).asJava
+ }
+
+ private def readAndFindData(meta: (String, Int, (Array[Byte], Array[Byte]), Long, Int),
+ value: Array[Byte]): Option[FineGrainBlocklet] = {
+ val bytes = FileReader.readByteArray(filePath, meta._4, meta._5)
+ val outputStream = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(outputStream)
+ val blockletsData = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]]
+
+ import scala.collection.Searching._
+ val searching = blockletsData
+ .search[(Array[Byte], Seq[Seq[Int]], Seq[Int])]((value, Seq(Seq(0)), Seq(0)))(new Ordering[
+ (Array[Byte], Seq[Seq[Int]], Seq[Int])] {
+ override def compare(x: (Array[Byte], Seq[Seq[Int]], Seq[Int]),
+ y: (Array[Byte], Seq[Seq[Int]], Seq[Int])) = {
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(x._1, y._1)
+ }
+ })
+ if (searching.insertionPoint >= 0) {
+ val f = blockletsData(searching.insertionPoint)
+ val pages = f._3.zipWithIndex.map { p =>
+ val pg = new FineGrainBlocklet.Page
+ pg.setPageId(p._1)
+ pg.setRowId(f._2(p._2).toArray)
+ pg
+ }
+ pages
+ Some(new FineGrainBlocklet(meta._1, meta._2.toString, pages.toList.asJava))
+ } else {
+ None
+ }
+
+ }
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) >= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) <= 0
+ }
+ tuples
+ }
+
+ def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear():Unit = {
+ ???
+ }
+
+ override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
+}
+
+class FGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segmentId: String, dataWriterPath: String, dataMapName: String)
+ extends AbstractDataMapWriter(identifier, segmentId, dataWriterPath) {
+
+ var currentBlockId: String = null
+ val fgwritepath = dataWriterPath + "/" + System.nanoTime() + ".datamap"
+ val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(fgwritepath, FileFactory.getFileType(fgwritepath))
+ val blockletList = new ArrayBuffer[(Array[Byte], Seq[Int], Seq[Int])]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]()
+ var position: Long = 0
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Seq[Int]], Seq[Int]) = null
+ var addedLast: Boolean = false
+ val blockletListUpdated = new ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]()
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3 ++ f._3)
+ addedLast = false
+ } else {
+ blockletListUpdated += oldValue
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletListUpdated += oldValue
+ }
+
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(blockletListUpdated)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ maxMin +=
+ ((currentBlockId + "", blockletId, (blockletListUpdated.head._1, blockletListUpdated.last
+ ._1), position, bytes.length))
+ position += bytes.length
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[(Array[Byte], Int)]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += ((newBytes, i))
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Int], Seq[Int]) = null
+ var addedLast: Boolean = false
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3)
+ addedLast = false
+ } else {
+ blockletList += oldValue
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletList += oldValue
+ }
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(fgwritepath)
+ }
+
+
+}
+
+class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test fg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ table.getAbsoluteTableIdentifier,
+ classOf[FGDataMapFactory].getName, "fgdatamap")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala
index 25bdf7b..c200b1b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala
@@ -28,14 +28,14 @@ import org.apache.spark.sql.{DataFrame, SaveMode}
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.dev.{DataMap, DataMapFactory, DataMapWriter}
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMap}
import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.schema.FilterType
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.Event
-import org.apache.carbondata.spark.testsuite.datamap.C2DataMapFactory
class InsertOverwriteConcurrentTest extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
private val executorService: ExecutorService = Executors.newFixedThreadPool(10)
@@ -164,7 +164,7 @@ object Global {
var overwriteRunning = false
}
-class WaitingDataMap() extends DataMapFactory {
+class WaitingDataMap() extends AbstractCoarseGrainDataMapFactory {
override def init(identifier: AbsoluteTableIdentifier, dataMapName: String): Unit = { }
@@ -174,12 +174,12 @@ class WaitingDataMap() extends DataMapFactory {
override def clear(): Unit = {}
- override def getDataMaps(distributable: DataMapDistributable): java.util.List[DataMap] = ???
+ override def getDataMaps(distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = ???
- override def getDataMaps(segmentId: String): util.List[DataMap] = ???
+ override def getDataMaps(segmentId: String): util.List[AbstractCoarseGrainDataMap] = ???
- override def createWriter(segmentId: String): DataMapWriter = {
- new DataMapWriter {
+ override def createWriter(segmentId: String, writerPath: String): AbstractDataMapWriter = {
+ new AbstractDataMapWriter(null, segmentId, writerPath) {
override def onPageAdded(blockletId: Int, pageId: Int, pages: Array[ColumnPage]): Unit = { }
override def onBlockletEnd(blockletId: Int): Unit = { }
@@ -195,10 +195,14 @@ class WaitingDataMap() extends DataMapFactory {
// wait for 1 second to let second SQL to finish
Thread.sleep(1000)
}
+
+ override def finish(): Unit = {
+
+ }
}
}
- override def getMeta: DataMapMeta = new DataMapMeta(List("o_country").asJava, FilterType.EQUALTO)
+ override def getMeta: DataMapMeta = new DataMapMeta(List("o_country").asJava, Seq(ExpressionType.EQUALS).asJava)
override def toDistributable(segmentId: String): util.List[DataMapDistributable] = ???
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 084a748..1b68458 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -438,9 +438,10 @@ class CarbonScanRDD(
CarbonTableInputFormat.setQuerySegment(conf, identifier)
CarbonTableInputFormat.setFilterPredicates(conf, filterExpression)
CarbonTableInputFormat.setColumnProjection(conf, columnProjection)
- if (CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
- CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT).toBoolean) {
+ CarbonTableInputFormat.setDataMapJob(conf, new SparkDataMapJob)
+ if (CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT).toBoolean) {
CarbonTableInputFormat.setDataMapJob(conf, new SparkDataMapJob)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index f5a90de..2552ca8 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -778,8 +778,9 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
.start()
qry.awaitTermination()
} catch {
- case ex =>
- throw new Exception(ex.getMessage)
+ case ex: Throwable =>
+ LOGGER.error(ex.getMessage)
+ throw new Exception(ex.getMessage, ex)
} finally {
if (null != qry) {
qry.stop()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 8e350d9..31a6701 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.processing.datamap;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -28,8 +29,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
import org.apache.carbondata.core.datamap.TableDataMap;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.processing.store.TablePage;
@@ -43,25 +44,26 @@ public class DataMapWriterListener {
DataMapWriterListener.class.getCanonicalName());
// list indexed column name -> list of data map writer
- private Map<List<String>, List<DataMapWriter>> registry = new ConcurrentHashMap<>();
+ private Map<List<String>, List<AbstractDataMapWriter>> registry = new ConcurrentHashMap<>();
/**
* register all datamap writer for specified table and segment
*/
- public void registerAllWriter(AbsoluteTableIdentifier identifier, String segmentId) {
+ public void registerAllWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String dataWritePath) {
List<TableDataMap> tableDataMaps = DataMapStoreManager.getInstance().getAllDataMap(identifier);
if (tableDataMaps != null) {
for (TableDataMap tableDataMap : tableDataMaps) {
DataMapFactory factory = tableDataMap.getDataMapFactory();
- register(factory, segmentId);
+ register(factory, segmentId, dataWritePath);
}
}
}
/**
- * Register a DataMapWriter
+ * Register a AbstractDataMapWriter
*/
- private void register(DataMapFactory factory, String segmentId) {
+ private void register(DataMapFactory factory, String segmentId, String dataWritePath) {
assert (factory != null);
assert (segmentId != null);
DataMapMeta meta = factory.getMeta();
@@ -70,8 +72,8 @@ public class DataMapWriterListener {
return;
}
List<String> columns = factory.getMeta().getIndexedColumns();
- List<DataMapWriter> writers = registry.get(columns);
- DataMapWriter writer = factory.createWriter(segmentId);
+ List<AbstractDataMapWriter> writers = registry.get(columns);
+ AbstractDataMapWriter writer = factory.createWriter(segmentId, dataWritePath);
if (writers != null) {
writers.add(writer);
} else {
@@ -79,36 +81,36 @@ public class DataMapWriterListener {
writers.add(writer);
registry.put(columns, writers);
}
- LOG.info("DataMapWriter " + writer + " added");
+ LOG.info("AbstractDataMapWriter " + writer + " added");
}
public void onBlockStart(String blockId, String blockPath) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
- writer.onBlockStart(blockId, blockPath);
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
+ writer.onBlockStart(blockId);
}
}
}
public void onBlockEnd(String blockId) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
writer.onBlockEnd(blockId);
}
}
}
public void onBlockletStart(int blockletId) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
writer.onBlockletStart(blockletId);
}
}
}
public void onBlockletEnd(int blockletId) {
- for (List<DataMapWriter> writers : registry.values()) {
- for (DataMapWriter writer : writers) {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
writer.onBlockletEnd(blockletId);
}
}
@@ -121,18 +123,29 @@ public class DataMapWriterListener {
* @param tablePage page data
*/
public void onPageAdded(int blockletId, int pageId, TablePage tablePage) {
- Set<Map.Entry<List<String>, List<DataMapWriter>>> entries = registry.entrySet();
- for (Map.Entry<List<String>, List<DataMapWriter>> entry : entries) {
+ Set<Map.Entry<List<String>, List<AbstractDataMapWriter>>> entries = registry.entrySet();
+ for (Map.Entry<List<String>, List<AbstractDataMapWriter>> entry : entries) {
List<String> indexedColumns = entry.getKey();
ColumnPage[] pages = new ColumnPage[indexedColumns.size()];
for (int i = 0; i < indexedColumns.size(); i++) {
pages[i] = tablePage.getColumnPage(indexedColumns.get(i));
}
- List<DataMapWriter> writers = entry.getValue();
- for (DataMapWriter writer : writers) {
+ List<AbstractDataMapWriter> writers = entry.getValue();
+ for (AbstractDataMapWriter writer : writers) {
writer.onPageAdded(blockletId, pageId, pages);
}
}
}
+ /**
+ * Finish all datamap writers
+ */
+ public void finish() throws IOException {
+ for (List<AbstractDataMapWriter> writers : registry.values()) {
+ for (AbstractDataMapWriter writer : writers) {
+ writer.finish();
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index bc87823..0e9cbc5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
@@ -259,7 +260,8 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.sortScope = CarbonDataProcessorUtil.getSortScope(configuration);
DataMapWriterListener listener = new DataMapWriterListener();
- listener.registerAllWriter(configuration.getTableIdentifier(), configuration.getSegmentId());
+ listener.registerAllWriter(configuration.getTableIdentifier(), configuration.getSegmentId(),
+ storeLocation[new Random().nextInt(storeLocation.length)]);
carbonFactDataHandlerModel.dataMapWriterlistener = listener;
carbonFactDataHandlerModel.writingCoresCount = configuration.getWritingCoresCount();
@@ -323,6 +325,12 @@ public class CarbonFactDataHandlerModel {
segmentProperties.getDimensions(),
segmentProperties.getMeasures());
+ DataMapWriterListener listener = new DataMapWriterListener();
+ listener.registerAllWriter(
+ loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier(),
+ loadModel.getSegmentId(),
+ tempStoreLocation[new Random().nextInt(tempStoreLocation.length)]);
+ carbonFactDataHandlerModel.dataMapWriterlistener = listener;
return carbonFactDataHandlerModel;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 7d0a285..f229c7a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.processing.store.writer;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
@@ -41,14 +39,11 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonMergerUtil;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.util.CarbonProperties;
@@ -63,7 +58,6 @@ import org.apache.carbondata.processing.datamap.DataMapWriterListener;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
import org.apache.commons.lang3.ArrayUtils;
-import org.apache.hadoop.io.IOUtils;
public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
@@ -71,12 +65,6 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
LogServiceFactory.getLogService(AbstractFactDataWriter.class.getName());
/**
- * dfs.bytes-per-checksum
- * HDFS checksum length, block size for a file should be exactly divisible
- * by this value
- */
- private static final int HDFS_CHECKSUM_LENGTH = 512;
- /**
* file channel
*/
protected FileChannel fileChannel;
@@ -208,35 +196,6 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
}
/**
- * This method will return max of block size and file size
- *
- * @param blockSize
- * @param fileSize
- * @return
- */
- private static long getMaxOfBlockAndFileSize(long blockSize, long fileSize) {
- long maxSize = blockSize;
- if (fileSize > blockSize) {
- maxSize = fileSize;
- }
- // block size should be exactly divisible by 512 which is maintained by HDFS as bytes
- // per checksum, dfs.bytes-per-checksum=512 must divide block size
- long remainder = maxSize % HDFS_CHECKSUM_LENGTH;
- if (remainder > 0) {
- maxSize = maxSize + HDFS_CHECKSUM_LENGTH - remainder;
- }
- // convert to make block size more readable.
- String readableBlockSize = ByteUtil.convertByteToReadable(blockSize);
- String readableFileSize = ByteUtil.convertByteToReadable(fileSize);
- String readableMaxSize = ByteUtil.convertByteToReadable(maxSize);
- LOGGER.info(
- "The configured block size is " + readableBlockSize + ", the actual carbon file size is "
- + readableFileSize + ", choose the max value " + readableMaxSize
- + " as the block size on HDFS");
- return maxSize;
- }
-
- /**
* This method will be used to update the file channel with new file if exceeding block size
* threshold, new file will be created once existing file reached the file size limit This
* method will first check whether existing file size is exceeded the file
@@ -282,7 +241,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
private String constructFactFileFullPath() {
String factFilePath =
- this.dataWriterVo.getCarbonDataDirectoryPath() + File.separator + this.carbonDataFileName;
+ this.model.getCarbonDataDirectoryPath() + File.separator + this.carbonDataFileName;
return factFilePath;
}
/**
@@ -293,7 +252,9 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
notifyDataMapBlockEnd();
CarbonUtil.closeStreams(this.fileOutputStream, this.fileChannel);
if (copyInCurrentThread) {
- copyCarbonDataFileToCarbonStorePath(carbonDataFileTempPath);
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(
+ carbonDataFileTempPath, model.getCarbonDataDirectoryPath(),
+ fileSizeInBytes);
} else {
executorServiceSubmitList.add(executorService.submit(new CopyThread(carbonDataFileTempPath)));
}
@@ -445,7 +406,9 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
}
writer.close();
// copy from temp to actual store location
- copyCarbonDataFileToCarbonStorePath(fileName);
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(fileName,
+ model.getCarbonDataDirectoryPath(),
+ fileSizeInBytes);
}
/**
@@ -455,80 +418,20 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
* @throws CarbonDataWriterException
*/
protected void closeExecutorService() throws CarbonDataWriterException {
- executorService.shutdown();
try {
+ listener.finish();
+ executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- throw new CarbonDataWriterException(e.getMessage());
- }
- for (int i = 0; i < executorServiceSubmitList.size(); i++) {
- try {
+ for (int i = 0; i < executorServiceSubmitList.size(); i++) {
executorServiceSubmitList.get(i).get();
- } catch (InterruptedException e) {
- throw new CarbonDataWriterException(e.getMessage());
- } catch (ExecutionException e) {
- throw new CarbonDataWriterException(e.getMessage());
}
+ } catch (InterruptedException | ExecutionException | IOException e) {
+ LOGGER.error(e, "Error while finishing writer");
+ throw new CarbonDataWriterException(e.getMessage());
}
}
- /**
- * This method will copy the given file to carbon store location
- *
- * @param localFileName local file name with full path
- * @throws CarbonDataWriterException
- */
- protected void copyCarbonDataFileToCarbonStorePath(String localFileName)
- throws CarbonDataWriterException {
- long copyStartTime = System.currentTimeMillis();
- LOGGER.info("Copying " + localFileName + " --> " + model.getCarbonDataDirectoryPath());
- try {
- CarbonFile localCarbonFile =
- FileFactory.getCarbonFile(localFileName, FileFactory.getFileType(localFileName));
- String carbonFilePath = model.getCarbonDataDirectoryPath() + localFileName
- .substring(localFileName.lastIndexOf(File.separator));
- copyLocalFileToCarbonStore(carbonFilePath, localFileName,
- CarbonCommonConstants.BYTEBUFFER_SIZE,
- getMaxOfBlockAndFileSize(fileSizeInBytes, localCarbonFile.getSize()));
- } catch (IOException e) {
- throw new CarbonDataWriterException(
- "Problem while copying file from local store to carbon store", e);
- }
- LOGGER.info(
- "Total copy time (ms) to copy file " + localFileName + " is " + (System.currentTimeMillis()
- - copyStartTime));
- }
-
- /**
- * This method will read the local carbon data file and write to carbon data file in HDFS
- *
- * @param carbonStoreFilePath
- * @param localFilePath
- * @param bufferSize
- * @param blockSize
- * @throws IOException
- */
- private void copyLocalFileToCarbonStore(String carbonStoreFilePath, String localFilePath,
- int bufferSize, long blockSize) throws IOException {
- DataOutputStream dataOutputStream = null;
- DataInputStream dataInputStream = null;
- try {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize
- + " (bytes");
- }
- dataOutputStream = FileFactory
- .getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
- bufferSize, blockSize);
- dataInputStream = FileFactory
- .getDataInputStream(localFilePath, FileFactory.getFileType(localFilePath), bufferSize);
- IOUtils.copyBytes(dataInputStream, dataOutputStream, bufferSize);
- } finally {
- CarbonUtil.closeStream(dataInputStream);
- CarbonUtil.closeStream(dataOutputStream);
- }
- }
/**
* This method will copy the carbon data file from local store location to
@@ -553,7 +456,10 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
* @throws Exception if unable to compute a result
*/
@Override public Void call() throws Exception {
- copyCarbonDataFileToCarbonStorePath(fileName);
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(
+ fileName,
+ model.getCarbonDataDirectoryPath(),
+ fileSizeInBytes);
return null;
}
[17/50] [abbrv] carbondata git commit:
[CARBONDATA-2075][CARBONDATA-1516] Fix test error with new creating
timeseries pre-aggregate table syntax
Posted by ra...@apache.org.
[CARBONDATA-2075][CARBONDATA-1516] Fix test error with new creating timeseries pre-aggregate table syntax
This closes #1890
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/15fabf9a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/15fabf9a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/15fabf9a
Branch: refs/heads/fgdatamap
Commit: 15fabf9ab6596e6f14c7d61065a19bbf2ae9af3f
Parents: 033870d
Author: xubo245 <60...@qq.com>
Authored: Wed Jan 31 10:49:08 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 19:07:31 2018 +0800
----------------------------------------------------------------------
.../preaggregate/TestPreAggCreateCommand.scala | 8 ++--
.../timeseries/TestTimeSeriesDropSuite.scala | 42 ++++++++++----------
.../testsuite/datamap/TestDataMapCommand.scala | 28 ++++++++++---
3 files changed, 47 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15fabf9a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index d3f0ff8..303abf4 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -217,7 +217,7 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
val timeSeries = TIMESERIES.toString
- test("test PreAggregate table selection: create with preaggregate and hierarchy") {
+ test("test pre agg create table 21: create with preaggregate and hierarchy") {
sql("DROP TABLE IF EXISTS maintabletime")
sql(
"""
@@ -252,8 +252,8 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS maintabletime")
}
- test("test pre agg create table 21: using") {
- sql("DROP DATAMAP agg0 ON TABLE maintable")
+ test("test pre agg create table 22: using invalid datamap provider") {
+ sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
val e: Exception = intercept[Exception] {
sql(
@@ -267,7 +267,7 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
}
assert(e.getMessage.contains(
s"Unknown data map type abc"))
- sql("DROP DATAMAP agg0 ON TABLE maintable")
+ sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
}
def getCarbontable(plan: LogicalPlan) : CarbonTable ={
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15fabf9a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
index f6d41fb..545c4de 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
@@ -28,10 +28,10 @@ class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll with Befo
sql(
"""
| CREATE TABLE mainTable(
- | dataTime timestamp,
- | name string,
- | city string,
- | age int)
+ | dataTime TIMESTAMP,
+ | name STRING,
+ | city STRING,
+ | age INT)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
}
@@ -42,46 +42,46 @@ class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll with Befo
val e: Exception = intercept[Exception] {
sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
}
- assert(e.getMessage.contains("Datamap with name agg1_month does not exist under table mainTable"))
+ assert(e.getMessage.equals("Datamap with name agg1_month does not exist under table mainTable"))
}
- test("test timeseries drop datamap 2: drop datamap should SUCCESS if haveIF EXISTS") {
+ test("test timeseries drop datamap 2: drop datamap should SUCCESS if have IF EXISTS") {
// DROP DATAMAP DataMapName if the DataMapName not exists
- checkExistence(sql("show datamap on table mainTable"), false, "agg1_month")
+ checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
try {
sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTable")
assert(true)
} catch {
- case e: Exception =>
- println(e)
+ case _: Exception =>
assert(false)
}
}
test("test timeseries drop datamap 3: drop datamap should throw proper exception") {
sql(
- """create datamap agg1 on table mainTable
- |using 'preaggregate'
- |DMPROPERTIES (
- | 'timeseries.eventTime'='dataTime',
- | 'timeseries.hierarchy'='month=1,year=1')
- |as select dataTime, sum(age) from mainTable
- |group by dataTime
+ """
+ | CREATE DATAMAP agg1_month ON TABLE mainTable
+ | USING 'timeseries'
+ | DMPROPERTIES (
+ | 'event_Time'='dataTime',
+ | 'month_granularity'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
""".stripMargin)
// Before DROP DATAMAP
- checkExistence(sql("show datamap on table mainTable"), true, "agg1_month", "agg1_year")
+ checkExistence(sql("show datamap on table mainTable"), true, "agg1_month")
// DROP DATAMAP DataMapName
sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
- checkExistence(sql("show datamap on table mainTable"), false, "agg1_month")
+ checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
val e: Exception = intercept[MalformedCarbonCommandException] {
sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
}
- assert(e.getMessage.contains("Datamap with name agg1_month does not exist under table mainTable"))
+ assert(e.getMessage.equals("Datamap with name agg1_month does not exist under table mainTable"))
}
- test("test timeseries drop datamap: drop datamap should throw exception if table not exist") {
+ test("test timeseries drop datamap 4: drop datamap should throw exception if table not exist") {
// DROP DATAMAP DataMapName if the DataMapName not exists and
checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
val e: Exception = intercept[Exception] {
@@ -91,7 +91,7 @@ class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll with Befo
"Dropping datamap agg1_month failed: Table or view 'maintablenotexist' not found "))
}
- test("test timeseries drop datamap: should throw exception if table not exist with IF EXISTS") {
+ test("test timeseries drop datamap 5: should throw exception if table not exist with IF EXISTS") {
// DROP DATAMAP DataMapName if the DataMapName not exists
// DROP DATAMAP should throw exception if table not exist, even though there is IF EXISTS"
checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15fabf9a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index 0860da1..146ad62 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -244,14 +244,30 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
test("create pre-agg table with path") {
sql("drop table if exists main_preagg")
sql("drop table if exists main ")
- val path = "./_pre-agg_test"
- sql("create table main(year int,month int,name string,salary int) stored by 'carbondata' tblproperties('sort_columns'='month,year,name')")
+ val warehouse = s"$metastoredb/warehouse"
+ val path = warehouse + "/" + System.nanoTime + "_preAggTestPath"
+ sql(
+ s"""
+ | create table main(
+ | year int,
+ | month int,
+ | name string,
+ | salary int)
+ | stored by 'carbondata'
+ | tblproperties('sort_columns'='month,year,name')
+ """.stripMargin)
sql("insert into main select 10,11,'amy',12")
sql("insert into main select 10,11,'amy',14")
- sql("create datamap preagg on table main " +
- "using 'preaggregate' " +
- s"dmproperties ('path'='$path') " +
- "as select name,avg(salary) from main group by name")
+ sql(
+ s"""
+ | create datamap preagg
+ | on table main
+ | using 'preaggregate'
+ | dmproperties ('path'='$path')
+ | as select name,avg(salary)
+ | from main
+ | group by name
+ """.stripMargin)
assertResult(true)(new File(path).exists())
assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
.list(new FilenameFilter {
[32/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 5ebf4cf..eb834a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -373,7 +373,6 @@ public abstract class AbstractDataFileFooterConverter {
cardinality[i] = segmentInfo.getColumn_cardinalities().get(i);
}
info.setColumnCardinality(cardinality);
- info.setNumberOfColumns(segmentInfo.getNum_cols());
return info;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index f04b28d..b16bc5e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -47,10 +47,10 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
@@ -80,7 +80,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.carbondata.core.reader.ThriftReader.TBaseCreator;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
@@ -243,16 +243,13 @@ public final class CarbonUtil {
public static ColumnGroupModel getColGroupModel(int[][] columnGroups) {
int[] columnSplit = new int[columnGroups.length];
int noOfColumnStore = columnSplit.length;
- boolean[] columnarStore = new boolean[noOfColumnStore];
for (int i = 0; i < columnGroups.length; i++) {
columnSplit[i] = columnGroups[i].length;
- columnarStore[i] = columnGroups[i].length <= 1;
}
ColumnGroupModel colGroupModel = new ColumnGroupModel();
colGroupModel.setNoOfColumnStore(noOfColumnStore);
colGroupModel.setColumnSplit(columnSplit);
- colGroupModel.setColumnarStore(columnarStore);
colGroupModel.setColumnGroup(columnGroups);
return colGroupModel;
}
@@ -413,7 +410,7 @@ public final class CarbonUtil {
}
}
- public static int getFirstIndexUsingBinarySearch(DimensionColumnDataChunk dimColumnDataChunk,
+ public static int getFirstIndexUsingBinarySearch(DimensionColumnPage dimColumnDataChunk,
int low, int high, byte[] compareValue, boolean matchUpLimit) {
int cmpResult = 0;
while (high >= low) {
@@ -452,7 +449,7 @@ public final class CarbonUtil {
* @return the compareValue's range index in the dimColumnDataChunk
*/
public static int[] getRangeIndexUsingBinarySearch(
- DimensionColumnDataChunk dimColumnDataChunk, int low, int high, byte[] compareValue) {
+ DimensionColumnPage dimColumnDataChunk, int low, int high, byte[] compareValue) {
int[] rangeIndex = new int[2];
int cmpResult = 0;
@@ -546,7 +543,7 @@ public final class CarbonUtil {
* @return index value
*/
public static int nextLesserValueToTarget(int currentIndex,
- DimensionColumnDataChunk dimColumnDataChunk, byte[] compareValue) {
+ DimensionColumnPage dimColumnDataChunk, byte[] compareValue) {
while (currentIndex - 1 >= 0
&& dimColumnDataChunk.compareTo(currentIndex - 1, compareValue) >= 0) {
--currentIndex;
@@ -566,7 +563,7 @@ public final class CarbonUtil {
* @return index value
*/
public static int nextGreaterValueToTarget(int currentIndex,
- DimensionColumnDataChunk dimColumnDataChunk, byte[] compareValue, int numerOfRows) {
+ DimensionColumnPage dimColumnDataChunk, byte[] compareValue, int numerOfRows) {
while (currentIndex + 1 < numerOfRows
&& dimColumnDataChunk.compareTo(currentIndex + 1, compareValue) <= 0) {
++currentIndex;
@@ -929,7 +926,7 @@ public final class CarbonUtil {
return false;
}
- public static boolean[] getDictionaryEncodingArray(QueryDimension[] queryDimensions) {
+ public static boolean[] getDictionaryEncodingArray(ProjectionDimension[] queryDimensions) {
boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
dictionaryEncodingArray[i] =
@@ -938,7 +935,7 @@ public final class CarbonUtil {
return dictionaryEncodingArray;
}
- public static boolean[] getDirectDictionaryEncodingArray(QueryDimension[] queryDimensions) {
+ public static boolean[] getDirectDictionaryEncodingArray(ProjectionDimension[] queryDimensions) {
boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
dictionaryEncodingArray[i] =
@@ -947,7 +944,7 @@ public final class CarbonUtil {
return dictionaryEncodingArray;
}
- public static boolean[] getImplicitColumnArray(QueryDimension[] queryDimensions) {
+ public static boolean[] getImplicitColumnArray(ProjectionDimension[] queryDimensions) {
boolean[] implicitColumnArray = new boolean[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
implicitColumnArray[i] = queryDimensions[i].getDimension().hasEncoding(Encoding.IMPLICIT);
@@ -955,7 +952,7 @@ public final class CarbonUtil {
return implicitColumnArray;
}
- public static boolean[] getComplexDataTypeArray(QueryDimension[] queryDimensions) {
+ public static boolean[] getComplexDataTypeArray(ProjectionDimension[] queryDimensions) {
boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
dictionaryEncodingArray[i] =
@@ -985,7 +982,6 @@ public final class CarbonUtil {
fileFooter.setColumnInTable(schema);
SegmentInfo segmentInfo = new SegmentInfo();
segmentInfo.setColumnCardinality(detailInfo.getDimLens());
- segmentInfo.setNumberOfColumns(detailInfo.getRowCount());
fileFooter.setSegmentInfo(segmentInfo);
return fileFooter;
}
@@ -1025,7 +1021,7 @@ public final class CarbonUtil {
* @return
*/
public static long calculateMetaSize(TableBlockInfo tableBlockInfo) throws IOException {
- FileHolder fileReader = null;
+ FileReader fileReader = null;
try {
long completeBlockLength = tableBlockInfo.getBlockLength();
long footerPointer = completeBlockLength - 8;
@@ -2156,19 +2152,19 @@ public final class CarbonUtil {
/**
* Below method will be used to check filter value is present in the data chunk or not
* @param filterValues
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param low
* @param high
* @param chunkRowIndex
* @return
*/
public static int isFilterPresent(byte[][] filterValues,
- DimensionColumnDataChunk dimensionColumnDataChunk, int low, int high, int chunkRowIndex) {
+ DimensionColumnPage dimensionColumnPage, int low, int high, int chunkRowIndex) {
int compareResult = 0;
int mid = 0;
while (low <= high) {
mid = (low + high) >>> 1;
- compareResult = dimensionColumnDataChunk.compareTo(chunkRowIndex, filterValues[mid]);
+ compareResult = dimensionColumnPage.compareTo(chunkRowIndex, filterValues[mid]);
if (compareResult < 0) {
high = mid - 1;
} else if (compareResult > 0) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
index e61b477..d665379 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
@@ -21,7 +21,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
@@ -45,7 +45,7 @@ public class DataFileFooterConverter extends AbstractDataFileFooterConverter {
@Override public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo)
throws IOException {
DataFileFooter dataFileFooter = new DataFileFooter();
- FileHolder fileReader = null;
+ FileReader fileReader = null;
try {
long completeBlockLength = tableBlockInfo.getBlockLength();
long footerPointer = completeBlockLength - 8;
@@ -123,7 +123,7 @@ public class DataFileFooterConverter extends AbstractDataFileFooterConverter {
}
@Override public List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException {
- FileHolder fileReader = null;
+ FileReader fileReader = null;
List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
try {
long completeBlockLength = tableBlockInfo.getBlockLength();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java
index 74fd09a..afacc0b 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java
@@ -36,17 +36,6 @@ public interface CarbonDictionaryWriter extends Closeable {
void write(String value) throws IOException;
/**
- * write method that accepts one value at a time
- * This method can be used when data is huge and memory is les. In that
- * case data can be stored to a file and an iterator can iterate over it and
- * pass one value at a time
- *
- * @param value unique dictionary value
- * @throws IOException if an I/O error occurs
- */
- void write(byte[] value) throws IOException;
-
- /**
* write method that accepts list of byte arrays as value
* This can be used when data is less, then string can be converted
* to byte array for each value and added to a list
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
index 82baccc..53411e9 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
@@ -144,7 +144,7 @@ public class CarbonDictionaryWriterImpl implements CarbonDictionaryWriter {
* @param value unique dictionary value
* @throws IOException if an I/O error occurs
*/
- @Override public void write(byte[] value) throws IOException {
+ private void write(byte[] value) throws IOException {
if (isFirstTime) {
init();
isFirstTime = false;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
index 8bf8a07..08dd791 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
@@ -87,7 +87,7 @@ public class DictionaryCacheLoaderImplTest {
return 9999;
}
};
- dictionaryCacheLoader.load(dictionaryInfo, columnIdentifier, 0L, 2L, true);
+ dictionaryCacheLoader.load(dictionaryInfo, 0L, 2L, true);
assertEquals(dictionaryInfo.getDictionaryChunks().getSize(), 4);
}
@@ -97,7 +97,7 @@ public class DictionaryCacheLoaderImplTest {
return 10000;
}
};
- dictionaryCacheLoader.load(dictionaryInfo, columnIdentifier, 0L, 2L, true);
+ dictionaryCacheLoader.load(dictionaryInfo, 0L, 2L, true);
assertEquals(dictionaryInfo.getDictionaryChunks().getSize(), 2);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
deleted file mode 100644
index 5c51c87..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import org.apache.carbondata.core.datastore.impl.DFSFileHolderImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-
-public class DFSFileHolderImplUnitTest {
-
- private static DFSFileHolderImpl dfsFileHolder;
- private static String fileName;
- private static String fileNameWithEmptyContent;
- private static File file;
- private static File fileWithEmptyContent;
-
- @BeforeClass public static void setup() {
- dfsFileHolder = new DFSFileHolderImpl();
- file = new File("Test.carbondata");
- fileWithEmptyContent = new File("TestEXception.carbondata");
-
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- if (!fileWithEmptyContent.exists()) try {
- fileWithEmptyContent.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream of = new FileOutputStream(file, true);
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
- br.write("Hello World");
- br.close();
- } catch (Exception e) {
- e.getMessage();
- }
- fileName = file.getAbsolutePath();
- fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
- }
-
- @AfterClass public static void tearDown() throws IOException {
- file.delete();
- fileWithEmptyContent.delete();
- dfsFileHolder.finish();
- }
-
- @Test public void testReadByteArray() throws IOException {
- byte[] result = dfsFileHolder.readByteArray(fileName, 1);
- byte[] expected_result = new byte[] { 72 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadByteArrayWithFilePath() throws IOException {
- byte[] result = dfsFileHolder.readByteArray(fileName, 2L, 2);
- byte[] expected_result = { 108, 108 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadLong() throws IOException {
- long actualResult = dfsFileHolder.readLong(fileName, 1L);
- long expectedResult = 7308335519855243122L;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test(expected = IOException.class)
- public void testReadLongForIoException() throws IOException {
- dfsFileHolder.readLong(fileNameWithEmptyContent, 1L);
- }
-
- @Test(expected = IOException.class)
- public void testReadIntForIoException() throws IOException{
- dfsFileHolder.readInt(fileNameWithEmptyContent, 1L);
- }
-
- @Test public void testReadInt() throws IOException {
- int actualResult = dfsFileHolder.readInt(fileName, 1L);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileName() throws IOException {
- int actualResult = dfsFileHolder.readInt(fileName);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test(expected = IOException.class)
- public void testReadIntWithFileNameForIOException() throws IOException {
- dfsFileHolder.readInt(fileNameWithEmptyContent);
- }
-
- @Test public void testDouble() throws IOException {
- double actualResult = dfsFileHolder.readDouble(fileName, 1L);
- double expectedResult = 7.3083355198552433E18;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test(expected = IOException.class)
- public void testDoubleForIoException() throws IOException {
- dfsFileHolder.readDouble(fileNameWithEmptyContent, 1L);
- }
-
- @Test
- public void testDoubleForIoExceptionwithUpdateCache() throws IOException {
- new MockUp<FileSystem>() {
- @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
- throws IOException {
- throw new IOException();
- }
-
- };
- dfsFileHolder.readDouble(fileName, 1L);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
new file mode 100644
index 0000000..da61a94
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.carbondata.core.datastore.impl.DFSFileReaderImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+public class DFSFileReaderImplUnitTest {
+
+ private static DFSFileReaderImpl dfsFileHolder;
+ private static String fileName;
+ private static String fileNameWithEmptyContent;
+ private static File file;
+ private static File fileWithEmptyContent;
+
+ @BeforeClass public static void setup() {
+ dfsFileHolder = new DFSFileReaderImpl();
+ file = new File("Test.carbondata");
+ fileWithEmptyContent = new File("TestEXception.carbondata");
+
+ if (!file.exists()) try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ if (!fileWithEmptyContent.exists()) try {
+ fileWithEmptyContent.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream of = new FileOutputStream(file, true);
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
+ br.write("Hello World");
+ br.close();
+ } catch (Exception e) {
+ e.getMessage();
+ }
+ fileName = file.getAbsolutePath();
+ fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
+ }
+
+ @AfterClass public static void tearDown() throws IOException {
+ file.delete();
+ fileWithEmptyContent.delete();
+ dfsFileHolder.finish();
+ }
+
+ @Test public void testReadByteArray() throws IOException {
+ byte[] result = dfsFileHolder.readByteArray(fileName, 1);
+ byte[] expected_result = new byte[] { 72 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadByteArrayWithFilePath() throws IOException {
+ byte[] result = dfsFileHolder.readByteArray(fileName, 2L, 2);
+ byte[] expected_result = { 108, 108 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadLong() throws IOException {
+ long actualResult = dfsFileHolder.readLong(fileName, 1L);
+ long expectedResult = 7308335519855243122L;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test(expected = IOException.class)
+ public void testReadLongForIoException() throws IOException {
+ dfsFileHolder.readLong(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test(expected = IOException.class)
+ public void testReadIntForIoException() throws IOException{
+ dfsFileHolder.readInt(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test public void testReadInt() throws IOException {
+ int actualResult = dfsFileHolder.readInt(fileName, 1L);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadIntWithFileName() throws IOException {
+ int actualResult = dfsFileHolder.readInt(fileName);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test(expected = IOException.class)
+ public void testReadIntWithFileNameForIOException() throws IOException {
+ dfsFileHolder.readInt(fileNameWithEmptyContent);
+ }
+
+ @Test public void testDouble() throws IOException {
+ double actualResult = dfsFileHolder.readDouble(fileName, 1L);
+ double expectedResult = 7.3083355198552433E18;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test(expected = IOException.class)
+ public void testDoubleForIoException() throws IOException {
+ dfsFileHolder.readDouble(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test
+ public void testDoubleForIoExceptionwithUpdateCache() throws IOException {
+ new MockUp<FileSystem>() {
+ @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
+ throws IOException {
+ throw new IOException();
+ }
+
+ };
+ dfsFileHolder.readDouble(fileName, 1L);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
deleted file mode 100644
index ed50d63..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import org.apache.carbondata.core.datastore.impl.FileHolderImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThat;
-
-public class FileHolderImplUnitTest {
-
- private static FileHolderImpl fileHolder;
- private static FileHolderImpl fileHolderWithCapacity;
- private static String fileName;
- private static String fileNameWithEmptyContent;
- private static File file;
- private static File fileWithEmptyContent;
-
- @BeforeClass public static void setup() {
- fileHolder = new FileHolderImpl();
- fileHolderWithCapacity = new FileHolderImpl(50);
- file = new File("Test.carbondata");
- fileWithEmptyContent = new File("TestEXception.carbondata");
-
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- if (!fileWithEmptyContent.exists()) try {
- fileWithEmptyContent.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream of = new FileOutputStream(file, true);
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
- br.write("Hello World");
- br.close();
- } catch (Exception e) {
- e.getMessage();
- }
- fileName = file.getAbsolutePath();
- fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
- }
-
- @AfterClass public static void tearDown() throws IOException {
- file.delete();
- fileWithEmptyContent.delete();
- fileHolder.finish();
- }
-
- @Test public void testReadByteArray() throws IOException {
- byte[] result = fileHolder.readByteArray(fileName, 1);
- byte[] expected_result = new byte[] { 72 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadByteArrayWithFilePath() throws IOException {
- byte[] result = fileHolder.readByteArray(fileName, 2L, 2);
- byte[] expected_result = { 108, 108 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadLong() throws IOException {
- long actualResult = fileHolder.readLong(fileName, 1L);
- long expectedResult = 7308335519855243122L;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadLongForIoException() throws IOException {
- fileHolder.readLong(fileNameWithEmptyContent, 1L);
- }
-
- @Test public void testReadIntForIoException() throws IOException {
- fileHolder.readInt(fileNameWithEmptyContent, 1L);
- }
-
- @Test public void testReadInt() throws IOException {
- int actualResult = fileHolder.readInt(fileName, 1L);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileName() throws IOException {
- int actualResult = fileHolder.readInt(fileName);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileNameForIOException() throws IOException {
- fileHolder.readInt(fileNameWithEmptyContent);
-
- }
-
- @Test public void testDouble() throws IOException {
- double actualResult = fileHolder.readDouble(fileName, 1L);
- double expectedResult = 7.3083355198552433E18;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testDoubleForIoException() throws IOException {
- fileHolder.readDouble(fileNameWithEmptyContent, 1L);
-
- }
-
- @Test public void testDoubleForIoExceptionwithUpdateCache() throws Exception {
- new MockUp<FileSystem>() {
- @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
- throws IOException {
- throw new IOException();
- }
-
- };
- try {
- fileHolder.readDouble(fileName, 1L);
- } catch (Exception e) {
- assertNull(e.getMessage());
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileReaderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileReaderImplUnitTest.java
new file mode 100644
index 0000000..a6d3235
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileReaderImplUnitTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.carbondata.core.datastore.impl.FileReaderImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+
+public class FileReaderImplUnitTest {
+
+ private static FileReaderImpl fileHolder;
+ private static FileReaderImpl fileHolderWithCapacity;
+ private static String fileName;
+ private static String fileNameWithEmptyContent;
+ private static File file;
+ private static File fileWithEmptyContent;
+
+ @BeforeClass public static void setup() {
+ fileHolder = new FileReaderImpl();
+ fileHolderWithCapacity = new FileReaderImpl(50);
+ file = new File("Test.carbondata");
+ fileWithEmptyContent = new File("TestEXception.carbondata");
+
+ if (!file.exists()) try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ if (!fileWithEmptyContent.exists()) try {
+ fileWithEmptyContent.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream of = new FileOutputStream(file, true);
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
+ br.write("Hello World");
+ br.close();
+ } catch (Exception e) {
+ e.getMessage();
+ }
+ fileName = file.getAbsolutePath();
+ fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
+ }
+
+ @AfterClass public static void tearDown() throws IOException {
+ file.delete();
+ fileWithEmptyContent.delete();
+ fileHolder.finish();
+ }
+
+ @Test public void testReadByteArray() throws IOException {
+ byte[] result = fileHolder.readByteArray(fileName, 1);
+ byte[] expected_result = new byte[] { 72 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadByteArrayWithFilePath() throws IOException {
+ byte[] result = fileHolder.readByteArray(fileName, 2L, 2);
+ byte[] expected_result = { 108, 108 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadLong() throws IOException {
+ long actualResult = fileHolder.readLong(fileName, 1L);
+ long expectedResult = 7308335519855243122L;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadLongForIoException() throws IOException {
+ fileHolder.readLong(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test public void testReadIntForIoException() throws IOException {
+ fileHolder.readInt(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test public void testReadInt() throws IOException {
+ int actualResult = fileHolder.readInt(fileName, 1L);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadIntWithFileName() throws IOException {
+ int actualResult = fileHolder.readInt(fileName);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadIntWithFileNameForIOException() throws IOException {
+ fileHolder.readInt(fileNameWithEmptyContent);
+
+ }
+
+ @Test public void testDouble() throws IOException {
+ double actualResult = fileHolder.readDouble(fileName, 1L);
+ double expectedResult = 7.3083355198552433E18;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testDoubleForIoException() throws IOException {
+ fileHolder.readDouble(fileNameWithEmptyContent, 1L);
+
+ }
+
+ @Test public void testDoubleForIoExceptionwithUpdateCache() throws Exception {
+ new MockUp<FileSystem>() {
+ @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
+ throws IOException {
+ throw new IOException();
+ }
+
+ };
+ try {
+ fileHolder.readDouble(fileName, 1L);
+ } catch (Exception e) {
+ assertNull(e.getMessage());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
index 9c43553..20036ec 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
@@ -84,7 +84,7 @@ public class SegmentPropertiesTest extends TestCase {
assertTrue(true);
}
- @Test public void testBlockMetadataHasProperDimensionBlockMapping() {
+ @Test public void testBlockMetadataHasProperDimensionChunkMapping() {
Map<Integer, Integer> dimensionOrdinalToBlockMapping = new HashMap<Integer, Integer>();
dimensionOrdinalToBlockMapping.put(0, 0);
dimensionOrdinalToBlockMapping.put(1, 1);
@@ -97,7 +97,7 @@ public class SegmentPropertiesTest extends TestCase {
dimensionOrdinalToBlockMapping.put(8, 5);
dimensionOrdinalToBlockMapping.put(9, 6);
Map<Integer, Integer> dimensionOrdinalToBlockMappingActual =
- blockMetadataInfos.getDimensionOrdinalToBlockMapping();
+ blockMetadataInfos.getDimensionOrdinalToChunkMapping();
assertEquals(dimensionOrdinalToBlockMapping.size(),
dimensionOrdinalToBlockMappingActual.size());
Iterator<Entry<Integer, Integer>> iterator =
@@ -112,12 +112,12 @@ public class SegmentPropertiesTest extends TestCase {
assertTrue(true);
}
- @Test public void testBlockMetadataHasProperMeasureBlockMapping() {
+ @Test public void testBlockMetadataHasProperMeasureChunkMapping() {
Map<Integer, Integer> measureOrdinalToBlockMapping = new HashMap<Integer, Integer>();
measureOrdinalToBlockMapping.put(0, 0);
measureOrdinalToBlockMapping.put(1, 1);
Map<Integer, Integer> measureOrdinalToBlockMappingActual =
- blockMetadataInfos.getMeasuresOrdinalToBlockMapping();
+ blockMetadataInfos.getMeasuresOrdinalToChunkMapping();
assertEquals(measureOrdinalToBlockMapping.size(), measureOrdinalToBlockMappingActual.size());
Iterator<Entry<Integer, Integer>> iterator = measureOrdinalToBlockMapping.entrySet().iterator();
while (iterator.hasNext()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
index bdb83cd..54b66a6 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
@@ -37,7 +37,7 @@ import org.junit.Test;
public class ColumnGroupDimensionDataChunkTest {
- static ColumnGroupDimensionDataChunk columnGroupDimensionDataChunk;
+ static ColumnGroupDimensionColumnPage columnGroupDimensionDataChunk;
static KeyGenerator keyGenerator;
@BeforeClass public static void setup() {
@@ -56,7 +56,7 @@ public class ColumnGroupDimensionDataChunkTest {
position += keyGenerator.getKeySizeInBytes();
}
columnGroupDimensionDataChunk =
- new ColumnGroupDimensionDataChunk(data, keyGenerator.getKeySizeInBytes(), 3);
+ new ColumnGroupDimensionColumnPage(data, keyGenerator.getKeySizeInBytes(), 3);
}
@Test public void fillChunkDataTest() {
@@ -64,7 +64,7 @@ public class ColumnGroupDimensionDataChunkTest {
ordinals.add(1);
KeyStructureInfo keyStructureInfo = getKeyStructureInfo(ordinals, keyGenerator);
byte[] buffer = new byte[1];
- columnGroupDimensionDataChunk.fillChunkData(buffer, 0, 1, keyStructureInfo);
+ columnGroupDimensionDataChunk.fillRawData(1, 0, buffer, keyStructureInfo);
assertEquals(buffer[0], 2);
}
@@ -81,7 +81,7 @@ public class ColumnGroupDimensionDataChunkTest {
ordinals.add(2);
KeyStructureInfo keyStructureInfo = getKeyStructureInfo(ordinals, keyGenerator);
keyStructureInfo.setMdkeyQueryDimensionOrdinal(new int[] { 2 });
- int res = columnGroupDimensionDataChunk.fillConvertedChunkData(2, 2, row, keyStructureInfo);
+ int res = columnGroupDimensionDataChunk.fillSurrogateKey(2, 2, row, keyStructureInfo);
Assert.assertTrue(Arrays.equals(row, expected));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
index 04b7a80..f327ef6 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
@@ -30,7 +30,7 @@ import org.junit.Test;
public class FixedLengthDimensionDataChunkTest {
- static FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk;
+ static FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
static byte[] data;
@BeforeClass public static void setup() {
@@ -40,7 +40,7 @@ public class FixedLengthDimensionDataChunkTest {
int invertedIndexReverse[] = { 1, 0, 5, 7, 8 };
fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(data, invertedIndex, invertedIndexReverse, 5, 4);
+ new FixedLengthDimensionColumnPage(data, invertedIndex, invertedIndexReverse, 5, 4);
}
@Test public void fillChunkDataTest() {
@@ -48,7 +48,7 @@ public class FixedLengthDimensionDataChunkTest {
int[] maskByteRanges = { 1, 2, 4, 6, 5 };
keyStructureInfo.setMaskByteRanges(maskByteRanges);
keyStructureInfo.setMaxKey("1234567".getBytes());
- int res = fixedLengthDimensionDataChunk.fillChunkData(data, 0, 0, keyStructureInfo);
+ int res = fixedLengthDimensionDataChunk.fillRawData(0, 0, data, keyStructureInfo);
int expectedResult = 4 ;
assertEquals(res, expectedResult);
}
@@ -62,7 +62,7 @@ public class FixedLengthDimensionDataChunkTest {
@Test public void fillConvertedChunkDataTest() {
int[] row = { 1, 2, 4, 6 };
KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
- int res = fixedLengthDimensionDataChunk.fillConvertedChunkData(1, 0, row, keyStructureInfo);
+ int res = fixedLengthDimensionDataChunk.fillSurrogateKey(1, 0, row, keyStructureInfo);
int expectedResult = 1;
assertEquals(res, expectedResult);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
index d874037..7f7719c 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
@@ -76,9 +76,9 @@ public class BTreeBlockFinderTest extends TestCase {
buffer.array();
IndexKey key = new IndexKey(null, buffer.array());
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(1, findFirstBlock.nodeNumber());
+ assertEquals(1, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(1, findLastBlock.nodeNumber());
+ assertEquals(1, findLastBlock.nodeIndex());
}
@Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary() {
@@ -96,9 +96,9 @@ public class BTreeBlockFinderTest extends TestCase {
buffer.array();
IndexKey key = new IndexKey(null, buffer.array());
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(0, findFirstBlock.nodeNumber());
+ assertEquals(0, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(0, findLastBlock.nodeNumber());
+ assertEquals(0, findLastBlock.nodeIndex());
}
@Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey1()
@@ -118,9 +118,9 @@ public class BTreeBlockFinderTest extends TestCase {
IndexKey key =
new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 1, 1 }), null);
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(0, findFirstBlock.nodeNumber());
+ assertEquals(0, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(0, findLastBlock.nodeNumber());
+ assertEquals(0, findLastBlock.nodeIndex());
}
@Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2()
@@ -141,9 +141,9 @@ public class BTreeBlockFinderTest extends TestCase {
new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 0, 0 }), null);
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(0, findFirstBlock.nodeNumber());
+ assertEquals(0, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(0, findLastBlock.nodeNumber());
+ assertEquals(0, findLastBlock.nodeIndex());
}
/**
@@ -169,9 +169,9 @@ public class BTreeBlockFinderTest extends TestCase {
new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 10001, 10001 }), null);
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(99, findFirstBlock.nodeNumber());
+ assertEquals(99, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(99, findLastBlock.nodeNumber());
+ assertEquals(99, findLastBlock.nodeIndex());
}
private List<DataFileFooter> getDataFileFooterList() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
index 85bdfd8..be91410 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
@@ -50,32 +50,32 @@ public class RawBasedResultCollectorTest {
// aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
// aggregatorInfo.setMeasureDataTypes(
// new DataType[] { DataTypes.INT, DataTypes.TIMESTAMP, DataTypes.INT, DataTypes.INT });
-// QueryMeasure queryMeasure1 = new QueryMeasure("QMCol1");
+// ProjectionMeasure queryMeasure1 = new ProjectionMeasure("QMCol1");
// queryMeasure1.setQueryOrder(1);
-// QueryMeasure queryMeasure2 = new QueryMeasure("QMCol2");
+// ProjectionMeasure queryMeasure2 = new ProjectionMeasure("QMCol2");
// queryMeasure1.setQueryOrder(2);
-// QueryMeasure queryMeasure3 = new QueryMeasure("QMCol3");
+// ProjectionMeasure queryMeasure3 = new ProjectionMeasure("QMCol3");
// queryMeasure1.setQueryOrder(3);
-// QueryMeasure queryMeasure4 = new QueryMeasure("QMCol4");
+// ProjectionMeasure queryMeasure4 = new ProjectionMeasure("QMCol4");
// queryMeasure1.setQueryOrder(4);
-// QueryDimension queryDimension1 = new QueryDimension("QDCol1");
+// ProjectionDimension queryDimension1 = new ProjectionDimension("QDCol1");
// queryDimension1.setQueryOrder(1);
// ColumnSchema columnSchema = new ColumnSchema();
// queryDimension1.setDimension(new CarbonDimension(columnSchema, 0, 0, 0, 0));
-// QueryDimension queryDimension2 = new QueryDimension("QDCol2");
+// ProjectionDimension queryDimension2 = new ProjectionDimension("QDCol2");
// queryDimension2.setQueryOrder(2);
// queryDimension2.setDimension(new CarbonDimension(columnSchema, 1, 1, 1, 1));
-// QueryDimension queryDimension3 = new QueryDimension("QDCol3");
+// ProjectionDimension queryDimension3 = new ProjectionDimension("QDCol3");
// queryDimension3.setQueryOrder(3);
// queryDimension3.setDimension(new CarbonDimension(columnSchema, 2, 0, 0, 0));
-// QueryDimension queryDimension4 = new QueryDimension("QDCol4");
+// ProjectionDimension queryDimension4 = new ProjectionDimension("QDCol4");
// queryDimension4.setQueryOrder(4);
// queryDimension4.setDimension(new CarbonDimension(columnSchema, 3, 0, 0, 0));
-// blockExecutionInfo.setQueryDimensions(
-// new QueryDimension[] { queryDimension1, queryDimension2, queryDimension3,
+// blockExecutionInfo.setProjectionDimensions(
+// new ProjectionDimension[] { queryDimension1, queryDimension2, queryDimension3,
// queryDimension4 });
-// blockExecutionInfo.setQueryMeasures(
-// new QueryMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
+// blockExecutionInfo.setProjectionMeasures(
+// new ProjectionMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
// blockExecutionInfo.setFixedKeyUpdateRequired(true);
// blockExecutionInfo.setMeasureInfo(aggregatorInfo);
// blockExecutionInfo.setMaskedByteForBlock(new int[] { 1, 2 });
@@ -99,7 +99,7 @@ public class RawBasedResultCollectorTest {
// return new byte[][] { { 1, 2 }, { 1, 2 } };
// }
//
-// @SuppressWarnings("unused") @Mock public ColumnPage getMeasureChunk(int ordinal) {
+// @SuppressWarnings("unused") @Mock public ColumnPage readMeasureChunk(int ordinal) {
// ColumnPage ColumnPage = new ColumnPage();
// PresenceMeta presenceMeta = new PresenceMeta();
// BitSet bitSet = new BitSet();
@@ -120,10 +120,10 @@ public class RawBasedResultCollectorTest {
// }
// };
//
-// AbstractScannedResult abstractScannedResult =
+// BlockletScannedResult abstractScannedResult =
// new NonFilterQueryScannedResult(blockExecutionInfo);
-// abstractScannedResult.setNumberOfRows(2);
-// List<Object[]> result = rawBasedResultCollector.collectData(abstractScannedResult, 2);
+// abstractScannedResult.setPageFilteredRowCount(2);
+// List<Object[]> result = rawBasedResultCollector.collectResultInRow(abstractScannedResult, 2);
// int expectedResult = 2;
// assertThat(result.size(), is(equalTo(expectedResult)));
// }
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
index 45e86f2..e8b75b7 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.SegmentPropertiesTestUtil;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import junit.framework.TestCase;
import mockit.Mock;
@@ -45,9 +45,8 @@ public class QueryUtilTest extends TestCase {
@Test public void testGetMaskedByteRangeGivingProperMaksedByteRange() {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(0).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(0));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(0));
int[] maskedByteRange = QueryUtil
.getMaskedByteRange(Arrays.asList(dimension), segmentProperties.getDimensionKeyGenerator());
int[] expectedMaskedByteRange = { 0 };
@@ -57,11 +56,10 @@ public class QueryUtilTest extends TestCase {
}
@Test public void testGetMaskedByteRangeGivingProperMaksedByteRangeOnlyForDictionaryKey() {
- List<QueryDimension> dimensions = new ArrayList<QueryDimension>();
+ List<ProjectionDimension> dimensions = new ArrayList<ProjectionDimension>();
for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(i));
dimensions.add(dimension);
}
int[] maskedByteRange =
@@ -84,11 +82,10 @@ public class QueryUtilTest extends TestCase {
}
@Test public void testGetMaxKeyBasedOnDimensions() {
- List<QueryDimension> dimensions = new ArrayList<QueryDimension>();
+ List<ProjectionDimension> dimensions = new ArrayList<ProjectionDimension>();
for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(i));
dimensions.add(dimension);
}
byte[] maxKeyBasedOnDimensions = null;
@@ -115,10 +112,8 @@ public class QueryUtilTest extends TestCase {
}
@Test public void testGetMaksedByte() {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(0).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(0));
- dimension.setDimension(segmentProperties.getDimensions().get(0));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(0));
int[] maskedByteRange = QueryUtil
.getMaskedByteRange(Arrays.asList(dimension), segmentProperties.getDimensionKeyGenerator());
int[] maskedByte = QueryUtil
@@ -203,18 +198,16 @@ public class QueryUtilTest extends TestCase {
}
@Test public void testGetSortDimensionIndexes() {
- List<QueryDimension> sortedDimensions = new ArrayList<QueryDimension>();
+ List<ProjectionDimension> sortedDimensions = new ArrayList<ProjectionDimension>();
for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(i));
sortedDimensions.add(dimension);
}
- List<QueryDimension> queryDimensions = new ArrayList<QueryDimension>();
+ List<ProjectionDimension> queryDimensions = new ArrayList<ProjectionDimension>();
for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(i));
queryDimensions.add(dimension);
}
byte[] actualValue = QueryUtil.getSortDimensionIndexes(sortedDimensions, queryDimensions);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
index 172a53e..bdf81e7 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
@@ -28,8 +28,8 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.junit.Test;
@@ -79,22 +79,20 @@ public class RestructureUtilTest {
List<CarbonDimension> tableComplexDimensions =
Arrays.asList(tableComplexDimension1, tableComplexDimension2);
- QueryDimension queryDimension1 = new QueryDimension("Id");
- queryDimension1.setDimension(tableBlockDimension1);
- QueryDimension queryDimension2 = new QueryDimension("Name");
- queryDimension2.setDimension(tableComplexDimension2);
- QueryDimension queryDimension3 = new QueryDimension("Address");
- queryDimension3.setDimension(new CarbonDimension(columnSchema5, 3, 3, 3, 3));
+ ProjectionDimension queryDimension1 = new ProjectionDimension(tableBlockDimension1);
+ ProjectionDimension queryDimension2 = new ProjectionDimension(tableComplexDimension2);
+ ProjectionDimension
+ queryDimension3 = new ProjectionDimension(new CarbonDimension(columnSchema5, 3, 3, 3, 3));
- List<QueryDimension> queryDimensions =
+ List<ProjectionDimension> queryDimensions =
Arrays.asList(queryDimension1, queryDimension2, queryDimension3);
- List<QueryDimension> result = null;
+ List<ProjectionDimension> result = null;
result = RestructureUtil
.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo, queryDimensions,
tableBlockDimensions, tableComplexDimensions);
List<CarbonDimension> resultDimension = new ArrayList<>(result.size());
- for (QueryDimension queryDimension : result) {
+ for (ProjectionDimension queryDimension : result) {
resultDimension.add(queryDimension.getDimension());
}
assertThat(resultDimension,
@@ -121,13 +119,10 @@ public class RestructureUtilTest {
carbonMeasure3.getColumnSchema().setDefaultValue("3".getBytes());
List<CarbonMeasure> currentBlockMeasures = Arrays.asList(carbonMeasure1, carbonMeasure2);
- QueryMeasure queryMeasure1 = new QueryMeasure("Id");
- queryMeasure1.setMeasure(carbonMeasure1);
- QueryMeasure queryMeasure2 = new QueryMeasure("Name");
- queryMeasure2.setMeasure(carbonMeasure2);
- QueryMeasure queryMeasure3 = new QueryMeasure("Age");
- queryMeasure3.setMeasure(carbonMeasure3);
- List<QueryMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2, queryMeasure3);
+ ProjectionMeasure queryMeasure1 = new ProjectionMeasure(carbonMeasure1);
+ ProjectionMeasure queryMeasure2 = new ProjectionMeasure(carbonMeasure2);
+ ProjectionMeasure queryMeasure3 = new ProjectionMeasure(carbonMeasure3);
+ List<ProjectionMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2, queryMeasure3);
BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
RestructureUtil.createMeasureInfoAndGetCurrentBlockQueryMeasures(blockExecutionInfo, queryMeasures,
currentBlockMeasures);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
index 89b3122..cbb939b 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -325,7 +325,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
return "test";
}
};
- assertTrue(FilterUtil.getFilterListForRS(expression, columnExpression, defaultValues,
+ assertTrue(FilterUtil.getFilterListForRS(expression, defaultValues,
defaultSurrogate) instanceof ColumnFilterInfo);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
index e3ae42c..11c4980 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
@@ -18,12 +18,12 @@ package org.apache.carbondata.core.scan.filter.executer;
import java.util.BitSet;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.util.CarbonUtil;
public class ExcludeFilterExecuterImplTest extends IncludeFilterExecuterImplTest {
- @Override public BitSet setFilterdIndexToBitSetNew(DimensionColumnDataChunk dimColumnDataChunk,
+ @Override public BitSet setFilterdIndexToBitSetNew(DimensionColumnPage dimColumnDataChunk,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
@@ -46,7 +46,7 @@ public class ExcludeFilterExecuterImplTest extends IncludeFilterExecuterImplTest
return bitSet;
}
- @Override public BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimColumnDataChunk,
+ @Override public BitSet setFilterdIndexToBitSet(DimensionColumnPage dimColumnDataChunk,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
index 29dda52..1bfa875 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
@@ -18,8 +18,8 @@ package org.apache.carbondata.core.scan.filter.executer;
import java.util.BitSet;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import org.apache.carbondata.core.util.CarbonUtil;
import org.junit.Assert;
@@ -38,22 +38,22 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
- public BitSet setFilterdIndexToBitSetNew(DimensionColumnDataChunk dimensionColumnDataChunk,
+ public BitSet setFilterdIndexToBitSetNew(DimensionColumnPage dimensionColumnPage,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
- if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ if (dimensionColumnPage instanceof FixedLengthDimensionColumnPage) {
// byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
if (filterValues.length > 1) {
for (int i = 0; i < numerOfRows; i++) {
int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
- dimensionColumnDataChunk.getChunkData(i));
+ dimensionColumnPage.getChunkData(i));
if (index >= 0) {
bitSet.set(i);
}
}
} else if (filterValues.length == 1) {
for (int i = 0; i < numerOfRows; i++) {
- if (dimensionColumnDataChunk.compareTo(i, filterValues[0]) == 0) {
+ if (dimensionColumnPage.compareTo(i, filterValues[0]) == 0) {
bitSet.set(i);
}
}
@@ -62,14 +62,14 @@ public class IncludeFilterExecuterImplTest extends TestCase {
return bitSet;
}
- public BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows,
+ public BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage, int numerOfRows,
byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
- if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ if (dimensionColumnPage instanceof FixedLengthDimensionColumnPage) {
// byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
for (int k = 0; k < filterValues.length; k++) {
for (int j = 0; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[k]) == 0) {
+ if (dimensionColumnPage.compareTo(j, filterValues[k]) == 0) {
bitSet.set(j);
}
}
@@ -163,7 +163,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
System.out.println("dimColumnSize: " + dimColumnSize);
- FixedLengthDimensionDataChunk dimensionColumnDataChunk;
+ FixedLengthDimensionColumnPage dimensionColumnDataChunk;
DimColumnExecuterFilterInfo dim = new DimColumnExecuterFilterInfo();
byte[] dataChunk = new byte[dataChunkSize * dimColumnSize];
@@ -183,7 +183,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
dim.setFilterKeys(filterKeys);
- dimensionColumnDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ dimensionColumnDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunkSize, dimColumnSize);
// repeat query and compare 2 result between old code and new optimized code
@@ -194,7 +194,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
oldTime = oldTime + System.currentTimeMillis() - start;
start = System.currentTimeMillis();
- BitSet bitNew = this.setFilterdIndexToBitSetNew((FixedLengthDimensionDataChunk) dimensionColumnDataChunk, dataChunkSize,
+ BitSet bitNew = this.setFilterdIndexToBitSetNew((FixedLengthDimensionColumnPage) dimensionColumnDataChunk, dataChunkSize,
filterKeys);
newTime = newTime + System.currentTimeMillis() - start;
@@ -212,7 +212,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
- private BitSet setFilterdIndexToBitSetWithColumnIndexOld(FixedLengthDimensionDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSetWithColumnIndexOld(FixedLengthDimensionColumnPage dimensionColumnDataChunk,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
@@ -243,7 +243,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
return bitSet;
}
- private BitSet setFilterdIndexToBitSetWithColumnIndexNew(FixedLengthDimensionDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSetWithColumnIndexNew(FixedLengthDimensionColumnPage dimensionColumnDataChunk,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
int startIndex = 0;
@@ -281,7 +281,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
int filteredValueCnt = 800;
// column dictionary size
int dimColumnSize = 2;
- FixedLengthDimensionDataChunk dimensionColumnDataChunk;
+ FixedLengthDimensionColumnPage dimensionColumnDataChunk;
DimColumnExecuterFilterInfo dim = new DimColumnExecuterFilterInfo();
byte[] dataChunk = new byte[dataChunkSize * dimColumnSize];
@@ -303,7 +303,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
dim.setFilterKeys(filterKeys);
- dimensionColumnDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ dimensionColumnDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / dimColumnSize, dimColumnSize);
// initial to run
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java b/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java
deleted file mode 100644
index 541ce48..0000000
--- a/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.result;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-public class BatchResultTest {
- private static BatchResult batchResult;
- private static List<Object[]> rowsList = new ArrayList(2);
-
- @BeforeClass public static void setUp() {
- batchResult = new BatchResult();
- rowsList.add(0, new Integer[] { 1, 2 });
- rowsList.add(1, new Integer[] { 3 });
- }
-
- @Test public void testNext() throws NoSuchElementException {
- BatchResult rows = new BatchResult();
- rows.setRows(rowsList);
- Object[] result = rows.next();
- Assert.assertTrue(result.equals(rowsList.get(0)));
- }
-
- @Test(expected = NoSuchElementException.class) public void testNextWithNoSuchElementException() {
- BatchResult rows = new BatchResult();
- List emptyList = new ArrayList(2);
- rows.setRows(emptyList);
- rows.next();
- }
-
- @Test public void testGetRows() {
- new MockUp<BatchResult>() {
- @Mock public void $init() {
- //to be left blank
- }
- };
- BatchResult batchResult = new BatchResult();
- List<Object[]> list = batchResult.getRows();
- assertNull("Number of rows is null", list);
- }
-
- @Test public void testHasNext() {
- List<Object[]> list = new ArrayList<>();
- list.add(0, new Integer[] { 1, 2 });
- list.add(1, new Integer[] { 1, 2 });
- batchResult.setRows(list);
- boolean result = batchResult.hasNext();
- Assert.assertTrue(result);
- }
-
- @Test public void testGetRawRow() {
- List<Object[]> list = new ArrayList<>();
- list.add(0, new Integer[] { 1, 2 });
- batchResult.setRows(list);
- Object[] actualValue = batchResult.getRawRow(0);
- Assert.assertTrue(list.get(0) == actualValue);
- }
-
- @Test public void testGetSize() {
- List<Object[]> list = new ArrayList<>();
- list.add(0, new Integer[] { 1, 2 });
- list.add(1, new Integer[] { 1, 2 });
- batchResult.setRows(list);
- int actualValue = batchResult.getSize();
- int expectedValue = 2;
- assertEquals(expectedValue, actualValue);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/test/java/org/apache/carbondata/core/scan/result/RowBatchTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/result/RowBatchTest.java b/core/src/test/java/org/apache/carbondata/core/scan/result/RowBatchTest.java
new file mode 100644
index 0000000..123d64e
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/result/RowBatchTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.result;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class RowBatchTest {
+ private static RowBatch rowBatch;
+ private static List<Object[]> rowsList = new ArrayList(2);
+
+ @BeforeClass public static void setUp() {
+ rowBatch = new RowBatch();
+ rowsList.add(0, new Integer[] { 1, 2 });
+ rowsList.add(1, new Integer[] { 3 });
+ }
+
+ @Test public void testNext() throws NoSuchElementException {
+ RowBatch rows = new RowBatch();
+ rows.setRows(rowsList);
+ Object[] result = rows.next();
+ Assert.assertTrue(result.equals(rowsList.get(0)));
+ }
+
+ @Test(expected = NoSuchElementException.class) public void testNextWithNoSuchElementException() {
+ RowBatch rows = new RowBatch();
+ List emptyList = new ArrayList(2);
+ rows.setRows(emptyList);
+ rows.next();
+ }
+
+ @Test public void testGetRows() {
+ new MockUp<RowBatch>() {
+ @Mock public void $init() {
+ //to be left blank
+ }
+ };
+ RowBatch rowBatch = new RowBatch();
+ List<Object[]> list = rowBatch.getRows();
+ assertNull("Number of rows is null", list);
+ }
+
+ @Test public void testHasNext() {
+ List<Object[]> list = new ArrayList<>();
+ list.add(0, new Integer[] { 1, 2 });
+ list.add(1, new Integer[] { 1, 2 });
+ rowBatch.setRows(list);
+ boolean result = rowBatch.hasNext();
+ Assert.assertTrue(result);
+ }
+
+ @Test public void testGetRawRow() {
+ List<Object[]> list = new ArrayList<>();
+ list.add(0, new Integer[] { 1, 2 });
+ rowBatch.setRows(list);
+ Object[] actualValue = rowBatch.getRawRow(0);
+ Assert.assertTrue(list.get(0) == actualValue);
+ }
+
+ @Test public void testGetSize() {
+ List<Object[]> list = new ArrayList<>();
+ list.add(0, new Integer[] { 1, 2 });
+ list.add(1, new Integer[] { 1, 2 });
+ rowBatch.setRows(list);
+ int actualValue = rowBatch.getSize();
+ int expectedValue = 2;
+ assertEquals(expectedValue, actualValue);
+ }
+
+}
[34/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
new file mode 100644
index 0000000..fde4e55
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.processor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.scan.collector.ResultCollectorFactory;
+import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.scanner.BlockletScanner;
+import org.apache.carbondata.core.scan.scanner.impl.BlockletFilterScanner;
+import org.apache.carbondata.core.scan.scanner.impl.BlockletFullScanner;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.util.TaskMetricsMap;
+
+/**
+ * This abstract class provides a skeletal implementation of the
+ * Block iterator.
+ */
+public class DataBlockIterator extends CarbonIterator<List<Object[]>> {
+
+ /**
+ * iterator which will be used to iterate over blocklets
+ */
+ private BlockletIterator blockletIterator;
+
+ /**
+ * result collector which will be used to aggregate the scanned result
+ */
+ private ScannedResultCollector scannerResultAggregator;
+
+ /**
+ * processor which will be used to process the block processing can be
+ * filter processing or non filter processing
+ */
+ private BlockletScanner blockletScanner;
+
+ /**
+ * batch size of result
+ */
+ private int batchSize;
+
+ private ExecutorService executorService;
+
+ private Future<BlockletScannedResult> future;
+
+ private Future<RawBlockletColumnChunks> futureIo;
+
+ private BlockletScannedResult scannedResult;
+
+ private BlockExecutionInfo blockExecutionInfo;
+
+ private FileReader fileReader;
+
+ private AtomicBoolean nextBlock;
+
+ private AtomicBoolean nextRead;
+
+ public DataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileReader fileReader,
+ int batchSize, QueryStatisticsModel queryStatisticsModel, ExecutorService executorService) {
+ this.blockExecutionInfo = blockExecutionInfo;
+ this.fileReader = fileReader;
+ blockletIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
+ blockExecutionInfo.getNumberOfBlockToScan());
+ if (blockExecutionInfo.getFilterExecuterTree() != null) {
+ blockletScanner = new BlockletFilterScanner(blockExecutionInfo, queryStatisticsModel);
+ } else {
+ blockletScanner = new BlockletFullScanner(blockExecutionInfo, queryStatisticsModel);
+ }
+ this.scannerResultAggregator =
+ ResultCollectorFactory.getScannedResultCollector(blockExecutionInfo);
+ this.batchSize = batchSize;
+ this.executorService = executorService;
+ this.nextBlock = new AtomicBoolean(false);
+ this.nextRead = new AtomicBoolean(false);
+ }
+
+ @Override
+ public List<Object[]> next() {
+ List<Object[]> collectedResult = null;
+ if (updateScanner()) {
+ collectedResult = this.scannerResultAggregator.collectResultInRow(scannedResult, batchSize);
+ while (collectedResult.size() < batchSize && updateScanner()) {
+ List<Object[]> data = this.scannerResultAggregator
+ .collectResultInRow(scannedResult, batchSize - collectedResult.size());
+ collectedResult.addAll(data);
+ }
+ } else {
+ collectedResult = new ArrayList<>();
+ }
+ return collectedResult;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (scannedResult != null && scannedResult.hasNext()) {
+ return true;
+ } else {
+ if (null != scannedResult) {
+ scannedResult.freeMemory();
+ }
+ return blockletIterator.hasNext() || nextBlock.get() || nextRead.get();
+ }
+ }
+
+ /**
+ * Return true if scan result if non-empty
+ */
+ private boolean updateScanner() {
+ try {
+ if (scannedResult != null && scannedResult.hasNext()) {
+ return true;
+ } else {
+ scannedResult = processNextBlocklet();
+ while (scannedResult != null) {
+ if (scannedResult.hasNext()) {
+ return true;
+ }
+ scannedResult = processNextBlocklet();
+ }
+ nextBlock.set(false);
+ nextRead.set(false);
+ return false;
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private BlockletScannedResult processNextBlocklet() throws Exception {
+ BlockletScannedResult result = null;
+ if (blockExecutionInfo.isPrefetchBlocklet()) {
+ if (blockletIterator.hasNext() || nextBlock.get() || nextRead.get()) {
+ if (future == null) {
+ future = scanNextBlockletAsync();
+ }
+ result = future.get();
+ nextBlock.set(false);
+ if (blockletIterator.hasNext() || nextRead.get()) {
+ nextBlock.set(true);
+ future = scanNextBlockletAsync();
+ }
+ }
+ } else {
+ if (blockletIterator.hasNext()) {
+ RawBlockletColumnChunks rawChunks = readNextBlockletColumnChunks();
+ if (rawChunks != null) {
+ result = blockletScanner.scanBlocklet(rawChunks);
+ }
+ }
+ }
+ return result;
+ }
+
+ private RawBlockletColumnChunks readNextBlockletColumnChunks() throws IOException {
+ RawBlockletColumnChunks rawBlockletColumnChunks = getNextBlockletColumnChunks();
+ if (rawBlockletColumnChunks != null) {
+ blockletScanner.readBlocklet(rawBlockletColumnChunks);
+ return rawBlockletColumnChunks;
+ }
+ return null;
+ }
+
+ private RawBlockletColumnChunks getNextBlockletColumnChunks() {
+ RawBlockletColumnChunks rawBlockletColumnChunks = null;
+ do {
+ DataRefNode dataBlock = blockletIterator.next();
+ if (dataBlock.getColumnsMaxValue() == null || blockletScanner.isScanRequired(dataBlock)) {
+ rawBlockletColumnChunks = RawBlockletColumnChunks.newInstance(
+ blockExecutionInfo.getTotalNumberDimensionToRead(),
+ blockExecutionInfo.getTotalNumberOfMeasureToRead(), fileReader, dataBlock);
+ }
+ } while (rawBlockletColumnChunks == null && blockletIterator.hasNext());
+ return rawBlockletColumnChunks;
+ }
+
+ private Future<BlockletScannedResult> scanNextBlockletAsync() {
+ return executorService.submit(new Callable<BlockletScannedResult>() {
+ @Override public BlockletScannedResult call() throws Exception {
+ if (futureIo == null) {
+ futureIo = readNextBlockletAsync();
+ }
+ RawBlockletColumnChunks rawBlockletColumnChunks = futureIo.get();
+ futureIo = null;
+ nextRead.set(false);
+ if (rawBlockletColumnChunks != null) {
+ if (blockletIterator.hasNext()) {
+ nextRead.set(true);
+ futureIo = readNextBlockletAsync();
+ }
+ return blockletScanner.scanBlocklet(rawBlockletColumnChunks);
+ }
+ return null;
+ }
+ });
+ }
+
+ private Future<RawBlockletColumnChunks> readNextBlockletAsync() {
+ return executorService.submit(new Callable<RawBlockletColumnChunks>() {
+ @Override public RawBlockletColumnChunks call() throws Exception {
+ try {
+ TaskMetricsMap.getInstance().registerThreadCallback();
+ if (blockletIterator.hasNext()) {
+ return readNextBlockletColumnChunks();
+ } else {
+ return null;
+ }
+ } finally {
+ // update read bytes metrics for this thread
+ TaskMetricsMap.getInstance().updateReadBytes(Thread.currentThread().getId());
+ }
+ }
+ });
+ }
+
+ public void processNextBatch(CarbonColumnarBatch columnarBatch) {
+ if (updateScanner()) {
+ this.scannerResultAggregator.collectResultInColumnarBatch(scannedResult, columnarBatch);
+ }
+ }
+
+
+ /**
+ * Close the resources
+ */
+ public void close() {
+ // free the current scanned result
+ if (null != scannedResult && !scannedResult.hasNext()) {
+ scannedResult.freeMemory();
+ }
+ // free any pre-fetched memory if present
+ if (null != future) {
+ try {
+ BlockletScannedResult blockletScannedResult = future.get();
+ if (blockletScannedResult != null) {
+ blockletScannedResult.freeMemory();
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/processor/RawBlockletColumnChunks.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/RawBlockletColumnChunks.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/RawBlockletColumnChunks.java
new file mode 100644
index 0000000..6b7e880
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/RawBlockletColumnChunks.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.processor;
+
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+/**
+ * Contains dimension and measure raw column chunks of one blocklet
+ */
+public class RawBlockletColumnChunks {
+
+ /**
+ * dimension column data chunk
+ */
+ private DimensionRawColumnChunk[] dimensionRawColumnChunks;
+
+ /**
+ * measure column data chunk
+ */
+ private MeasureRawColumnChunk[] measureRawColumnChunks;
+
+ /**
+ * file reader which will use to read the block from file
+ */
+ private FileReader fileReader;
+
+ /**
+ * data block
+ */
+ private DataRefNode dataBlock;
+
+ private BitSetGroup bitSetGroup;
+
+ private RawBlockletColumnChunks() { }
+
+ public static RawBlockletColumnChunks newInstance(int numberOfDimensionChunk,
+ int numberOfMeasureChunk, FileReader fileReader, DataRefNode dataBlock) {
+ RawBlockletColumnChunks instance = new RawBlockletColumnChunks();
+ instance.dimensionRawColumnChunks = new DimensionRawColumnChunk[numberOfDimensionChunk];
+ instance.measureRawColumnChunks = new MeasureRawColumnChunk[numberOfMeasureChunk];
+ instance.fileReader = fileReader;
+ instance.dataBlock = dataBlock;
+ return instance;
+ }
+
+ /**
+ * @return the dimensionRawColumnChunks
+ */
+ public DimensionRawColumnChunk[] getDimensionRawColumnChunks() {
+ return dimensionRawColumnChunks;
+ }
+
+ /**
+ * @param dimensionRawColumnChunks the dimensionRawColumnChunks to set
+ */
+ public void setDimensionRawColumnChunks(DimensionRawColumnChunk[] dimensionRawColumnChunks) {
+ this.dimensionRawColumnChunks = dimensionRawColumnChunks;
+ }
+
+ /**
+ * @return the measureRawColumnChunks
+ */
+ public MeasureRawColumnChunk[] getMeasureRawColumnChunks() {
+ return measureRawColumnChunks;
+ }
+
+ /**
+ * @param measureRawColumnChunks the measureRawColumnChunks to set
+ */
+ public void setMeasureRawColumnChunks(MeasureRawColumnChunk[] measureRawColumnChunks) {
+ this.measureRawColumnChunks = measureRawColumnChunks;
+ }
+
+ /**
+ * @return the fileReader
+ */
+ public FileReader getFileReader() {
+ return fileReader;
+ }
+
+ /**
+ * @return the dataBlock
+ */
+ public DataRefNode getDataBlock() {
+ return dataBlock;
+ }
+
+ public BitSetGroup getBitSetGroup() {
+ return bitSetGroup;
+ }
+
+ public void setBitSetGroup(BitSetGroup bitSetGroup) {
+ this.bitSetGroup = bitSetGroup;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
deleted file mode 100644
index 1c97725..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.processor.impl;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-
-/**
- * Below class will be used to process the block for detail query
- */
-public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
- /**
- * DataBlockIteratorImpl Constructor
- *
- * @param blockExecutionInfo execution information
- */
- public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
- int batchSize, QueryStatisticsModel queryStatisticsModel, ExecutorService executorService) {
- super(blockExecutionInfo, fileReader, batchSize, queryStatisticsModel, executorService);
- }
-
- /**
- * It scans the block and returns the result with @batchSize
- *
- * @return Result of @batchSize
- */
- public List<Object[]> next() {
- List<Object[]> collectedResult = null;
- if (updateScanner()) {
- collectedResult = this.scannerResultAggregator.collectData(scannedResult, batchSize);
- while (collectedResult.size() < batchSize && updateScanner()) {
- List<Object[]> data = this.scannerResultAggregator
- .collectData(scannedResult, batchSize - collectedResult.size());
- collectedResult.addAll(data);
- }
- } else {
- collectedResult = new ArrayList<>();
- }
- return collectedResult;
- }
-
- public void processNextBatch(CarbonColumnarBatch columnarBatch) {
- if (updateScanner()) {
- this.scannerResultAggregator.collectVectorBatch(scannedResult, columnarBatch);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
deleted file mode 100644
index b089fad..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
+++ /dev/null
@@ -1,698 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.result;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.Map;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
-import org.apache.carbondata.core.mutate.DeleteDeltaVo;
-import org.apache.carbondata.core.mutate.TupleIdEnum;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-
-/**
- * Scanned result class which will store and provide the result on request
- */
-public abstract class AbstractScannedResult {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractScannedResult.class.getName());
- /**
- * current row number
- */
- protected int currentRow = -1;
-
- protected int pageCounter;
- /**
- * row mapping indexes
- */
- protected int[][] rowMapping;
- /**
- * key size of the fixed length column
- */
- private int fixedLengthKeySize;
- /**
- * total number of rows per page
- */
- private int[] numberOfRows;
-
- /**
- * Total number of rows.
- */
- private int totalNumberOfRows;
- /**
- * to keep track of number of rows process
- */
- protected int rowCounter;
- /**
- * dimension column data chunk
- */
- protected DimensionColumnDataChunk[][] dimensionDataChunks;
-
- /**
- * Raw dimension chunks;
- */
- protected DimensionRawColumnChunk[] dimRawColumnChunks;
-
- /**
- * Raw dimension chunks;
- */
- protected MeasureRawColumnChunk[] msrRawColumnChunks;
- /**
- * measure column data chunk
- */
- protected ColumnPage[][] measureDataChunks;
- /**
- * dictionary column block index in file
- */
- protected int[] dictionaryColumnBlockIndexes;
-
- /**
- * no dictionary column block index in file
- */
- protected int[] noDictionaryColumnBlockIndexes;
-
- /**
- * column group to is key structure info
- * which will be used to get the key from the complete
- * column group key
- * For example if only one dimension of the column group is selected
- * then from complete column group key it will be used to mask the key and
- * get the particular column key
- */
- protected Map<Integer, KeyStructureInfo> columnGroupKeyStructureInfo;
-
- /**
- *
- */
- private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
-
- private int totalDimensionsSize;
-
- /**
- * blockedId which will be blockId + blocklet number in the block
- */
- private String blockletId;
-
- private long rowId;
-
- /**
- * parent block indexes
- */
- private int[] complexParentBlockIndexes;
-
- /**
- * blockletid+pageumber to deleted reocrd map
- */
- private Map<String, DeleteDeltaVo> deletedRecordMap;
-
- /**
- * current page delete delta vo
- */
- private DeleteDeltaVo currentDeleteDeltaVo;
-
- /**
- * actual blocklet number
- */
- private String blockletNumber;
-
- public AbstractScannedResult(BlockExecutionInfo blockExecutionInfo) {
- this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
- this.noDictionaryColumnBlockIndexes = blockExecutionInfo.getNoDictionaryBlockIndexes();
- this.dictionaryColumnBlockIndexes = blockExecutionInfo.getDictionaryColumnBlockIndex();
- this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
- this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
- this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
- this.totalDimensionsSize = blockExecutionInfo.getQueryDimensions().length;
- this.deletedRecordMap = blockExecutionInfo.getDeletedRecordsMap();
- }
-
- /**
- * Below method will be used to set the dimension chunks
- * which will be used to create a row
- *
- * @param dataChunks dimension chunks used in query
- */
- public void setDimensionChunks(DimensionColumnDataChunk[][] dataChunks) {
- this.dimensionDataChunks = dataChunks;
- }
-
- /**
- * Below method will be used to set the measure column chunks
- *
- * @param measureDataChunks measure data chunks
- */
- public void setMeasureChunks(ColumnPage[][] measureDataChunks) {
- this.measureDataChunks = measureDataChunks;
- }
-
- public void setDimRawColumnChunks(DimensionRawColumnChunk[] dimRawColumnChunks) {
- this.dimRawColumnChunks = dimRawColumnChunks;
- }
-
- public void setMsrRawColumnChunks(MeasureRawColumnChunk[] msrRawColumnChunks) {
- this.msrRawColumnChunks = msrRawColumnChunks;
- }
-
- /**
- * Below method will be used to get the chunk based in measure ordinal
- *
- * @param ordinal measure ordinal
- * @return measure column chunk
- */
- public ColumnPage getMeasureChunk(int ordinal) {
- return measureDataChunks[ordinal][pageCounter];
- }
-
- /**
- * Below method will be used to get the key for all the dictionary dimensions
- * which is present in the query
- *
- * @param rowId row id selected after scanning
- * @return return the dictionary key
- */
- protected byte[] getDictionaryKeyArray(int rowId) {
- byte[] completeKey = new byte[fixedLengthKeySize];
- int offset = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- offset += dimensionDataChunks[dictionaryColumnBlockIndexes[i]][pageCounter]
- .fillChunkData(completeKey, offset, rowId,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- rowCounter++;
- return completeKey;
- }
-
- /**
- * Below method will be used to get the key for all the dictionary dimensions
- * in integer array format which is present in the query
- *
- * @param rowId row id selected after scanning
- * @return return the dictionary key
- */
- protected int[] getDictionaryKeyIntegerArray(int rowId) {
- int[] completeKey = new int[totalDimensionsSize];
- int column = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[dictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(rowId, column, completeKey,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- rowCounter++;
- return completeKey;
- }
-
- /**
- * Fill the column data of dictionary to vector
- */
- public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[dictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(vectorInfo, column,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- }
-
- /**
- * Fill the column data to vector
- */
- public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(vectorInfo, column,
- columnGroupKeyStructureInfo.get(noDictionaryColumnBlockIndexes[i]));
- }
- }
-
- /**
- * Fill the measure column data to vector
- */
- public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
- for (int i = 0; i < measuresOrdinal.length; i++) {
- vectorInfo[i].measureVectorFiller
- .fillMeasureVector(measureDataChunks[measuresOrdinal[i]][pageCounter], vectorInfo[i]);
- }
- }
-
- public void fillColumnarComplexBatch(ColumnVectorInfo[] vectorInfos) {
- for (int i = 0; i < vectorInfos.length; i++) {
- int offset = vectorInfos[i].offset;
- int len = offset + vectorInfos[i].size;
- int vectorOffset = vectorInfos[i].vectorOffset;
- CarbonColumnVector vector = vectorInfos[i].vector;
- for (int j = offset; j < len; j++) {
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutput = new DataOutputStream(byteStream);
- try {
- vectorInfos[i].genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks,
- rowMapping == null ? j : rowMapping[pageCounter][j], pageCounter, dataOutput);
- Object data = vectorInfos[i].genericQueryType
- .getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
- vector.putObject(vectorOffset++, data);
- } catch (IOException e) {
- LOGGER.error(e);
- } finally {
- CarbonUtil.closeStreams(dataOutput);
- CarbonUtil.closeStreams(byteStream);
- }
- }
- }
- }
-
- /**
- * Fill the column data to vector
- */
- public void fillColumnarImplicitBatch(ColumnVectorInfo[] vectorInfo) {
- for (int i = 0; i < vectorInfo.length; i++) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[i];
- CarbonColumnVector vector = columnVectorInfo.vector;
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- for (int j = offset; j < len; j++) {
- // Considering only String case now as we support only
- String data = getBlockletId();
- if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
- .equals(columnVectorInfo.dimension.getColumnName())) {
- data = data + CarbonCommonConstants.FILE_SEPARATOR + pageCounter
- + CarbonCommonConstants.FILE_SEPARATOR + (rowMapping == null ?
- j :
- rowMapping[pageCounter][j]);
- }
- vector.putBytes(vectorOffset++,
- data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
- }
- }
- }
-
- /**
- * Just increment the counter incase of query only on measures.
- */
- public void incrementCounter() {
- rowCounter++;
- currentRow++;
- }
-
- /**
- * Just increment the page counter and reset the remaining counters.
- */
- public void incrementPageCounter() {
- rowCounter = 0;
- currentRow = -1;
- pageCounter++;
- fillDataChunks();
- if (null != deletedRecordMap) {
- currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
- }
- }
-
- /**
- * This case is used only in case of compaction, since it does not use filter flow.
- */
- public void fillDataChunks() {
- freeDataChunkMemory();
- if (pageCounter >= numberOfRows.length) {
- return;
- }
- for (int i = 0; i < dimensionDataChunks.length; i++) {
- if (dimensionDataChunks[i][pageCounter] == null && dimRawColumnChunks[i] != null) {
- dimensionDataChunks[i][pageCounter] =
- dimRawColumnChunks[i].convertToDimColDataChunkWithOutCache(pageCounter);
- }
- }
-
- for (int i = 0; i < measureDataChunks.length; i++) {
- if (measureDataChunks[i][pageCounter] == null && msrRawColumnChunks[i] != null) {
- measureDataChunks[i][pageCounter] =
- msrRawColumnChunks[i].convertToColumnPageWithOutCache(pageCounter);
- }
- }
- }
-
- // free the memory for the last page chunk
- private void freeDataChunkMemory() {
- for (int i = 0; i < dimensionDataChunks.length; i++) {
- if (pageCounter > 0 && dimensionDataChunks[i][pageCounter - 1] != null) {
- dimensionDataChunks[i][pageCounter - 1].freeMemory();
- dimensionDataChunks[i][pageCounter - 1] = null;
- }
- }
- for (int i = 0; i < measureDataChunks.length; i++) {
- if (pageCounter > 0 && measureDataChunks[i][pageCounter - 1] != null) {
- measureDataChunks[i][pageCounter - 1].freeMemory();
- measureDataChunks[i][pageCounter - 1] = null;
- }
- }
- }
-
- public int numberOfpages() {
- return numberOfRows.length;
- }
-
- /**
- * Get total rows in the current page
- *
- * @return
- */
- public int getCurrentPageRowCount() {
- return numberOfRows[pageCounter];
- }
-
- public int getCurrentPageCounter() {
- return pageCounter;
- }
-
- /**
- * increment the counter.
- */
- public void setRowCounter(int rowCounter) {
- this.rowCounter = rowCounter;
- }
-
- /**
- * Below method will be used to get the dimension data based on dimension
- * ordinal and index
- *
- * @param dimOrdinal dimension ordinal present in the query
- * @param rowId row index
- * @return dimension data based on row id
- */
- protected byte[] getDimensionData(int dimOrdinal, int rowId) {
- return dimensionDataChunks[dimOrdinal][pageCounter].getChunkData(rowId);
- }
-
- /**
- * Below method will be used to get the dimension key array
- * for all the no dictionary dimension present in the query
- *
- * @param rowId row number
- * @return no dictionary keys for all no dictionary dimension
- */
- protected byte[][] getNoDictionaryKeyArray(int rowId) {
- byte[][] noDictionaryColumnsKeys = new byte[noDictionaryColumnBlockIndexes.length][];
- int position = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- noDictionaryColumnsKeys[position++] =
- dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId);
- }
- return noDictionaryColumnsKeys;
- }
-
- /**
- * Below method will be used to get the dimension key array
- * for all the no dictionary dimension present in the query
- *
- * @param rowId row number
- * @return no dictionary keys for all no dictionary dimension
- */
- protected String[] getNoDictionaryKeyStringArray(int rowId) {
- String[] noDictionaryColumnsKeys = new String[noDictionaryColumnBlockIndexes.length];
- int position = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- noDictionaryColumnsKeys[position++] = new String(
- dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId),
- Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
- }
- return noDictionaryColumnsKeys;
- }
-
- /**
- * @return blockletId
- */
- public String getBlockletId() {
- return blockletId;
- }
-
- /**
- * @param blockletId
- */
- public void setBlockletId(String blockletId) {
- this.blockletId = CarbonTablePath.getShortBlockId(blockletId);
- blockletNumber = CarbonUpdateUtil.getRequiredFieldFromTID(blockletId, TupleIdEnum.BLOCKLET_ID);
- // if deleted recors map is present for this block
- // then get the first page deleted vo
- if (null != deletedRecordMap) {
- currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + '_' + pageCounter);
- }
- }
-
- /**
- * @return blockletId
- */
- public long getRowId() {
- return rowId;
- }
-
- /**
- * @param rowId
- */
- public void setRowId(long rowId) {
- this.rowId = rowId;
- }
-
- /**
- * Below method will be used to get the complex type keys array based
- * on row id for all the complex type dimension selected in query
- *
- * @param rowId row number
- * @return complex type key array for all the complex dimension selected in query
- */
- protected byte[][] getComplexTypeKeyArray(int rowId) {
- byte[][] complexTypeData = new byte[complexParentBlockIndexes.length][];
- for (int i = 0; i < complexTypeData.length; i++) {
- GenericQueryType genericQueryType =
- complexParentIndexToQueryMap.get(complexParentBlockIndexes[i]);
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutput = new DataOutputStream(byteStream);
- try {
- genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, rowId, pageCounter,
- dataOutput);
- complexTypeData[i] = byteStream.toByteArray();
- } catch (IOException e) {
- LOGGER.error(e);
- } finally {
- CarbonUtil.closeStreams(dataOutput);
- CarbonUtil.closeStreams(byteStream);
- }
- }
- return complexTypeData;
- }
-
- /**
- * @return return the total number of row after scanning
- */
- public int numberOfOutputRows() {
- return this.totalNumberOfRows;
- }
-
- /**
- * to check whether any more row is present in the result
- *
- * @return
- */
- public boolean hasNext() {
- if (pageCounter < numberOfRows.length && rowCounter < this.numberOfRows[pageCounter]) {
- return true;
- } else if (pageCounter < numberOfRows.length) {
- pageCounter++;
- fillDataChunks();
- rowCounter = 0;
- currentRow = -1;
- if (null != deletedRecordMap) {
- currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
- }
- return hasNext();
- }
- return false;
- }
-
- /**
- * Below method will be used to free the occupied memory
- */
- public void freeMemory() {
- // first free the dimension chunks
- if (null != dimensionDataChunks) {
- for (int i = 0; i < dimensionDataChunks.length; i++) {
- if (null != dimensionDataChunks[i]) {
- for (int j = 0; j < dimensionDataChunks[i].length; j++) {
- if (null != dimensionDataChunks[i][j]) {
- dimensionDataChunks[i][j].freeMemory();
- }
- }
- }
- }
- }
- // free the measure data chunks
- if (null != measureDataChunks) {
- for (int i = 0; i < measureDataChunks.length; i++) {
- if (null != measureDataChunks[i]) {
- for (int j = 0; j < measureDataChunks[i].length; j++) {
- if (null != measureDataChunks[i][j]) {
- measureDataChunks[i][j].freeMemory();
- }
- }
- }
- }
- }
- // free the raw chunks
- if (null != dimRawColumnChunks) {
- for (int i = 0; i < dimRawColumnChunks.length; i++) {
- if (null != dimRawColumnChunks[i]) {
- dimRawColumnChunks[i].freeMemory();
- }
- }
- }
- }
-
- /**
- * As this class will be a flyweight object so
- * for one block all the blocklet scanning will use same result object
- * in that case we need to reset the counter to zero so
- * for new result it will give the result from zero
- */
- public void reset() {
- rowCounter = 0;
- currentRow = -1;
- pageCounter = 0;
- }
-
- /**
- * @param numberOfRows set total of number rows valid after scanning
- */
- public void setNumberOfRows(int[] numberOfRows) {
- this.numberOfRows = numberOfRows;
-
- for (int count : numberOfRows) {
- totalNumberOfRows += count;
- }
- }
-
- /**
- * After applying filter it will return the bit set with the valid row indexes
- * so below method will be used to set the row indexes
- *
- * @param indexes
- */
- public void setIndexes(int[][] indexes) {
- this.rowMapping = indexes;
- }
-
- public int getRowCounter() {
- return rowCounter;
- }
-
- /**
- * will return the current valid row id
- *
- * @return valid row id
- */
- public abstract int getCurrentRowId();
-
- /**
- * @return dictionary key array for all the dictionary dimension
- * selected in query
- */
- public abstract byte[] getDictionaryKeyArray();
-
- /**
- * @return dictionary key array for all the dictionary dimension in integer array forat
- * selected in query
- */
- public abstract int[] getDictionaryKeyIntegerArray();
-
- /**
- * Below method will be used to get the complex type key array
- *
- * @return complex type key array
- */
- public abstract byte[][] getComplexTypeKeyArray();
-
- /**
- * Below method will be used to get the no dictionary key
- * array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- public abstract byte[][] getNoDictionaryKeyArray();
-
- /**
- * Below method will be used to get the no dictionary key
- * array in string array format for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- public abstract String[] getNoDictionaryKeyStringArray();
-
- /**
- * Mark the filtered rows in columnar batch. These rows will not be added to vector batches later.
- * @param columnarBatch
- * @param startRow
- * @param size
- * @param vectorOffset
- */
- public int markFilteredRows(CarbonColumnarBatch columnarBatch, int startRow, int size,
- int vectorOffset) {
- int rowsFiltered = 0;
- if (currentDeleteDeltaVo != null) {
- int len = startRow + size;
- for (int i = startRow; i < len; i++) {
- int rowId = rowMapping != null ? rowMapping[pageCounter][i] : i;
- if (currentDeleteDeltaVo.containsRow(rowId)) {
- columnarBatch.markFiltered(vectorOffset);
- rowsFiltered++;
- }
- vectorOffset++;
- }
- }
- return rowsFiltered;
- }
-
- /**
- * Below method will be used to check row got deleted
- *
- * @param rowId
- * @return is present in deleted row
- */
- public boolean containsDeletedRow(int rowId) {
- if (null != currentDeleteDeltaVo) {
- return currentDeleteDeltaVo.containsRow(rowId);
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java
deleted file mode 100644
index 56ca2ac..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.result;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import org.apache.carbondata.common.CarbonIterator;
-
-/**
- * Below class holds the query result
- */
-public class BatchResult extends CarbonIterator<Object[]> {
-
- /**
- * list of keys
- */
- protected List<Object[]> rows;
-
- /**
- * counter to check whether all the records are processed or not
- */
- protected int counter;
-
- public BatchResult() {
- this.rows = new ArrayList<>();
- }
-
- /**
- * Below method will be used to get the rows
- *
- * @return
- */
- public List<Object[]> getRows() {
- return rows;
- }
-
- /**
- * Below method will be used to get the set the values
- *
- * @param rows
- */
- public void setRows(List<Object[]> rows) {
- this.rows = rows;
- }
-
- /**
- * This method will return one row at a time based on the counter given.
- * @param counter
- * @return
- */
- public Object[] getRawRow(int counter) {
- return rows.get(counter);
- }
-
- /**
- * For getting the total size.
- * @return
- */
- public int getSize() {
- return rows.size();
- }
-
-
- /**
- * Returns {@code true} if the iteration has more elements.
- *
- * @return {@code true} if the iteration has more elements
- */
- @Override public boolean hasNext() {
- return counter < rows.size();
- }
-
- /**
- * Returns the next element in the iteration.
- *
- * @return the next element in the iteration
- */
- @Override public Object[] next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- Object[] row = rows.get(counter);
- counter++;
- return row;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
new file mode 100644
index 0000000..29404b4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -0,0 +1,618 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.result;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.mutate.DeleteDeltaVo;
+import org.apache.carbondata.core.mutate.TupleIdEnum;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+/**
+ * Scanned result class which will store and provide the result on request
+ */
+public abstract class BlockletScannedResult {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(BlockletScannedResult.class.getName());
+ /**
+ * current row number
+ */
+ protected int currentRow = -1;
+
+ protected int pageCounter;
+ /**
+ * matched rowId for each page
+ */
+ protected int[][] pageFilteredRowId;
+ /**
+ * key size of the fixed length column
+ */
+ private int fixedLengthKeySize;
+ /**
+ * total number of filtered rows for each page
+ */
+ private int[] pageFilteredRowCount;
+
+ /**
+ * to keep track of number of rows process
+ */
+ protected int rowCounter;
+ /**
+ * dimension column data chunk
+ */
+ protected DimensionColumnPage[][] dimensionColumnPages;
+
+ /**
+ * Raw dimension chunks;
+ */
+ protected DimensionRawColumnChunk[] dimRawColumnChunks;
+
+ /**
+ * Raw dimension chunks;
+ */
+ protected MeasureRawColumnChunk[] msrRawColumnChunks;
+ /**
+ * measure column data chunk
+ */
+ protected ColumnPage[][] measureColumnPages;
+ /**
+ * dictionary column block index in file
+ */
+ protected int[] dictionaryColumnChunkIndexes;
+
+ /**
+ * no dictionary column chunk index in file
+ */
+ protected int[] noDictionaryColumnChunkIndexes;
+
+ /**
+ * column group to is key structure info
+ * which will be used to get the key from the complete
+ * column group key
+ * For example if only one dimension of the column group is selected
+ * then from complete column group key it will be used to mask the key and
+ * get the particular column key
+ */
+ protected Map<Integer, KeyStructureInfo> columnGroupKeyStructureInfo;
+
+ /**
+ *
+ */
+ private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
+
+ private int totalDimensionsSize;
+
+ /**
+ * blockedId which will be blockId + blocklet number in the block
+ */
+ private String blockletId;
+
+ /**
+ * parent block indexes
+ */
+ private int[] complexParentBlockIndexes;
+
+ /**
+ * blockletid+pageumber to deleted reocrd map
+ */
+ private Map<String, DeleteDeltaVo> deletedRecordMap;
+
+ /**
+ * current page delete delta vo
+ */
+ private DeleteDeltaVo currentDeleteDeltaVo;
+
+ /**
+ * actual blocklet number
+ */
+ private String blockletNumber;
+
+ public BlockletScannedResult(BlockExecutionInfo blockExecutionInfo) {
+ this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
+ this.noDictionaryColumnChunkIndexes = blockExecutionInfo.getNoDictionaryColumnChunkIndexes();
+ this.dictionaryColumnChunkIndexes = blockExecutionInfo.getDictionaryColumnChunkIndex();
+ this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
+ this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
+ this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
+ this.totalDimensionsSize = blockExecutionInfo.getProjectionDimensions().length;
+ this.deletedRecordMap = blockExecutionInfo.getDeletedRecordsMap();
+ }
+
+ /**
+ * Below method will be used to set the dimension chunks
+ * which will be used to create a row
+ *
+ * @param columnPages dimension chunks used in query
+ */
+ public void setDimensionColumnPages(DimensionColumnPage[][] columnPages) {
+ this.dimensionColumnPages = columnPages;
+ }
+
+ /**
+ * Below method will be used to set the measure column chunks
+ *
+ * @param columnPages measure data chunks
+ */
+ public void setMeasureColumnPages(ColumnPage[][] columnPages) {
+ this.measureColumnPages = columnPages;
+ }
+
+ public void setDimRawColumnChunks(DimensionRawColumnChunk[] dimRawColumnChunks) {
+ this.dimRawColumnChunks = dimRawColumnChunks;
+ }
+
+ public void setMsrRawColumnChunks(MeasureRawColumnChunk[] msrRawColumnChunks) {
+ this.msrRawColumnChunks = msrRawColumnChunks;
+ }
+
+ /**
+ * Below method will be used to get the chunk based in measure ordinal
+ *
+ * @param ordinal measure ordinal
+ * @return measure column chunk
+ */
+ public ColumnPage getMeasureChunk(int ordinal) {
+ return measureColumnPages[ordinal][pageCounter];
+ }
+
+ /**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected byte[] getDictionaryKeyArray(int rowId) {
+ byte[] completeKey = new byte[fixedLengthKeySize];
+ int offset = 0;
+ for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
+ offset += dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter].fillRawData(
+ rowId, offset, completeKey,
+ columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * in integer array format which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected int[] getDictionaryKeyIntegerArray(int rowId) {
+ int[] completeKey = new int[totalDimensionsSize];
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
+ column = dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter]
+ .fillSurrogateKey(rowId, column, completeKey,
+ columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
+ * Fill the column data of dictionary to vector
+ */
+ public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
+ column = dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter]
+ .fillVector(vectorInfo, column,
+ columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the column data to vector
+ */
+ public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.noDictionaryColumnChunkIndexes.length; i++) {
+ column = dimensionColumnPages[noDictionaryColumnChunkIndexes[i]][pageCounter]
+ .fillVector(vectorInfo, column,
+ columnGroupKeyStructureInfo.get(noDictionaryColumnChunkIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the measure column data to vector
+ */
+ public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
+ for (int i = 0; i < measuresOrdinal.length; i++) {
+ vectorInfo[i].measureVectorFiller
+ .fillMeasureVector(measureColumnPages[measuresOrdinal[i]][pageCounter], vectorInfo[i]);
+ }
+ }
+
+ public void fillColumnarComplexBatch(ColumnVectorInfo[] vectorInfos) {
+ for (int i = 0; i < vectorInfos.length; i++) {
+ int offset = vectorInfos[i].offset;
+ int len = offset + vectorInfos[i].size;
+ int vectorOffset = vectorInfos[i].vectorOffset;
+ CarbonColumnVector vector = vectorInfos[i].vector;
+ for (int j = offset; j < len; j++) {
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutput = new DataOutputStream(byteStream);
+ try {
+ vectorInfos[i].genericQueryType.parseBlocksAndReturnComplexColumnByteArray(
+ dimRawColumnChunks,
+ pageFilteredRowId == null ? j : pageFilteredRowId[pageCounter][j], pageCounter,
+ dataOutput);
+ Object data = vectorInfos[i].genericQueryType
+ .getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
+ vector.putObject(vectorOffset++, data);
+ } catch (IOException e) {
+ LOGGER.error(e);
+ } finally {
+ CarbonUtil.closeStreams(dataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
+ }
+ }
+ }
+
+ /**
+ * Fill the column data to vector
+ */
+ public void fillColumnarImplicitBatch(ColumnVectorInfo[] vectorInfo) {
+ for (int i = 0; i < vectorInfo.length; i++) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[i];
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ for (int j = offset; j < len; j++) {
+ // Considering only String case now as we support only
+ String data = getBlockletId();
+ if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
+ .equals(columnVectorInfo.dimension.getColumnName())) {
+ data = data + CarbonCommonConstants.FILE_SEPARATOR + pageCounter
+ + CarbonCommonConstants.FILE_SEPARATOR + (pageFilteredRowId == null ?
+ j :
+ pageFilteredRowId[pageCounter][j]);
+ }
+ vector.putBytes(vectorOffset++,
+ data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+ }
+ }
+ }
+
+ /**
+ * Just increment the counter incase of query only on measures.
+ */
+ public void incrementCounter() {
+ rowCounter++;
+ currentRow++;
+ }
+
+ /**
+ * Just increment the page counter and reset the remaining counters.
+ */
+ public void incrementPageCounter() {
+ rowCounter = 0;
+ currentRow = -1;
+ pageCounter++;
+ fillDataChunks();
+ if (null != deletedRecordMap) {
+ currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
+ }
+ }
+
+ /**
+ * This case is used only in case of compaction, since it does not use filter flow.
+ */
+ public void fillDataChunks() {
+ freeDataChunkMemory();
+ if (pageCounter >= pageFilteredRowCount.length) {
+ return;
+ }
+ for (int i = 0; i < dimensionColumnPages.length; i++) {
+ if (dimensionColumnPages[i][pageCounter] == null && dimRawColumnChunks[i] != null) {
+ dimensionColumnPages[i][pageCounter] =
+ dimRawColumnChunks[i].convertToDimColDataChunkWithOutCache(pageCounter);
+ }
+ }
+
+ for (int i = 0; i < measureColumnPages.length; i++) {
+ if (measureColumnPages[i][pageCounter] == null && msrRawColumnChunks[i] != null) {
+ measureColumnPages[i][pageCounter] =
+ msrRawColumnChunks[i].convertToColumnPageWithOutCache(pageCounter);
+ }
+ }
+ }
+
+ // free the memory for the last page chunk
+ private void freeDataChunkMemory() {
+ for (int i = 0; i < dimensionColumnPages.length; i++) {
+ if (pageCounter > 0 && dimensionColumnPages[i][pageCounter - 1] != null) {
+ dimensionColumnPages[i][pageCounter - 1].freeMemory();
+ dimensionColumnPages[i][pageCounter - 1] = null;
+ }
+ }
+ for (int i = 0; i < measureColumnPages.length; i++) {
+ if (pageCounter > 0 && measureColumnPages[i][pageCounter - 1] != null) {
+ measureColumnPages[i][pageCounter - 1].freeMemory();
+ measureColumnPages[i][pageCounter - 1] = null;
+ }
+ }
+ }
+
+ public int numberOfpages() {
+ return pageFilteredRowCount.length;
+ }
+
+ /**
+ * Get total rows in the current page
+ *
+ * @return
+ */
+ public int getCurrentPageRowCount() {
+ return pageFilteredRowCount[pageCounter];
+ }
+
+ public int getCurrentPageCounter() {
+ return pageCounter;
+ }
+
+ /**
+ * increment the counter.
+ */
+ public void setRowCounter(int rowCounter) {
+ this.rowCounter = rowCounter;
+ }
+
+ /**
+ * Below method will be used to get the dimension key array
+ * for all the no dictionary dimension present in the query
+ *
+ * @param rowId row number
+ * @return no dictionary keys for all no dictionary dimension
+ */
+ protected byte[][] getNoDictionaryKeyArray(int rowId) {
+ byte[][] noDictionaryColumnsKeys = new byte[noDictionaryColumnChunkIndexes.length][];
+ int position = 0;
+ for (int i = 0; i < this.noDictionaryColumnChunkIndexes.length; i++) {
+ noDictionaryColumnsKeys[position++] =
+ dimensionColumnPages[noDictionaryColumnChunkIndexes[i]][pageCounter].getChunkData(rowId);
+ }
+ return noDictionaryColumnsKeys;
+ }
+
+ /**
+ * @return blockletId
+ */
+ public String getBlockletId() {
+ return blockletId;
+ }
+
+ /**
+ * Set blocklet id, which looks like
+ * "Part0/Segment_0/part-0-0_batchno0-0-1517155583332.carbondata/0"
+ */
+ public void setBlockletId(String blockletId) {
+ this.blockletId = CarbonTablePath.getShortBlockId(blockletId);
+ blockletNumber = CarbonUpdateUtil.getRequiredFieldFromTID(blockletId, TupleIdEnum.BLOCKLET_ID);
+ // if deleted recors map is present for this block
+ // then get the first page deleted vo
+ if (null != deletedRecordMap) {
+ currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + '_' + pageCounter);
+ }
+ }
+
+ /**
+ * Below method will be used to get the complex type keys array based
+ * on row id for all the complex type dimension selected in query
+ *
+ * @param rowId row number
+ * @return complex type key array for all the complex dimension selected in query
+ */
+ protected byte[][] getComplexTypeKeyArray(int rowId) {
+ byte[][] complexTypeData = new byte[complexParentBlockIndexes.length][];
+ for (int i = 0; i < complexTypeData.length; i++) {
+ GenericQueryType genericQueryType =
+ complexParentIndexToQueryMap.get(complexParentBlockIndexes[i]);
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutput = new DataOutputStream(byteStream);
+ try {
+ genericQueryType
+ .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, rowId, pageCounter,
+ dataOutput);
+ complexTypeData[i] = byteStream.toByteArray();
+ } catch (IOException e) {
+ LOGGER.error(e);
+ } finally {
+ CarbonUtil.closeStreams(dataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
+ }
+ return complexTypeData;
+ }
+
+ /**
+ * to check whether any more row is present in the result
+ *
+ * @return
+ */
+ public boolean hasNext() {
+ if (pageCounter
+ < pageFilteredRowCount.length && rowCounter < this.pageFilteredRowCount[pageCounter]) {
+ return true;
+ } else if (pageCounter < pageFilteredRowCount.length) {
+ pageCounter++;
+ fillDataChunks();
+ rowCounter = 0;
+ currentRow = -1;
+ if (null != deletedRecordMap) {
+ currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
+ }
+ return hasNext();
+ }
+ return false;
+ }
+
+ /**
+ * Below method will be used to free the occupied memory
+ */
+ public void freeMemory() {
+ // first free the dimension chunks
+ if (null != dimensionColumnPages) {
+ for (int i = 0; i < dimensionColumnPages.length; i++) {
+ if (null != dimensionColumnPages[i]) {
+ for (int j = 0; j < dimensionColumnPages[i].length; j++) {
+ if (null != dimensionColumnPages[i][j]) {
+ dimensionColumnPages[i][j].freeMemory();
+ }
+ }
+ }
+ }
+ }
+ // free the measure data chunks
+ if (null != measureColumnPages) {
+ for (int i = 0; i < measureColumnPages.length; i++) {
+ if (null != measureColumnPages[i]) {
+ for (int j = 0; j < measureColumnPages[i].length; j++) {
+ if (null != measureColumnPages[i][j]) {
+ measureColumnPages[i][j].freeMemory();
+ }
+ }
+ }
+ }
+ }
+ // free the raw chunks
+ if (null != dimRawColumnChunks) {
+ for (int i = 0; i < dimRawColumnChunks.length; i++) {
+ if (null != dimRawColumnChunks[i]) {
+ dimRawColumnChunks[i].freeMemory();
+ }
+ }
+ }
+ }
+
+ /**
+ * @param pageFilteredRowCount set total of number rows valid after scanning
+ */
+ public void setPageFilteredRowCount(int[] pageFilteredRowCount) {
+ this.pageFilteredRowCount = pageFilteredRowCount;
+ }
+
+ /**
+ * After applying filter it will return the bit set with the valid row indexes
+ * so below method will be used to set the row indexes
+ */
+ public void setPageFilteredRowId(int[][] pageFilteredRowId) {
+ this.pageFilteredRowId = pageFilteredRowId;
+ }
+
+ public int getRowCounter() {
+ return rowCounter;
+ }
+
+ /**
+ * will return the current valid row id
+ *
+ * @return valid row id
+ */
+ public abstract int getCurrentRowId();
+
+ /**
+ * @return dictionary key array for all the dictionary dimension
+ * selected in query
+ */
+ public abstract byte[] getDictionaryKeyArray();
+
+ /**
+ * @return dictionary key array for all the dictionary dimension in integer array forat
+ * selected in query
+ */
+ public abstract int[] getDictionaryKeyIntegerArray();
+
+ /**
+ * Below method will be used to get the complex type key array
+ *
+ * @return complex type key array
+ */
+ public abstract byte[][] getComplexTypeKeyArray();
+
+ /**
+ * Below method will be used to get the no dictionary key
+ * array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ public abstract byte[][] getNoDictionaryKeyArray();
+
+ /**
+ * Mark the filtered rows in columnar batch. These rows will not be added to vector batches later.
+ * @param columnarBatch
+ * @param startRow
+ * @param size
+ * @param vectorOffset
+ */
+ public int markFilteredRows(CarbonColumnarBatch columnarBatch, int startRow, int size,
+ int vectorOffset) {
+ int rowsFiltered = 0;
+ if (currentDeleteDeltaVo != null) {
+ int len = startRow + size;
+ for (int i = startRow; i < len; i++) {
+ int rowId = pageFilteredRowId != null ? pageFilteredRowId[pageCounter][i] : i;
+ if (currentDeleteDeltaVo.containsRow(rowId)) {
+ columnarBatch.markFiltered(vectorOffset);
+ rowsFiltered++;
+ }
+ vectorOffset++;
+ }
+ }
+ return rowsFiltered;
+ }
+
+ /**
+ * Below method will be used to check row got deleted
+ *
+ * @param rowId
+ * @return is present in deleted row
+ */
+ public boolean containsDeletedRow(int rowId) {
+ if (null != currentDeleteDeltaVo) {
+ return currentDeleteDeltaVo.containsRow(rowId);
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/RowBatch.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/RowBatch.java b/core/src/main/java/org/apache/carbondata/core/scan/result/RowBatch.java
new file mode 100644
index 0000000..c129161
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/RowBatch.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.result;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.carbondata.common.CarbonIterator;
+
+/**
+ * Below class holds the query result
+ */
+public class RowBatch extends CarbonIterator<Object[]> {
+
+ /**
+ * list of keys
+ */
+ protected List<Object[]> rows;
+
+ /**
+ * counter to check whether all the records are processed or not
+ */
+ protected int counter;
+
+ public RowBatch() {
+ this.rows = new ArrayList<>();
+ }
+
+ /**
+ * Below method will be used to get the rows
+ *
+ * @return
+ */
+ public List<Object[]> getRows() {
+ return rows;
+ }
+
+ /**
+ * Below method will be used to get the set the values
+ *
+ * @param rows
+ */
+ public void setRows(List<Object[]> rows) {
+ this.rows = rows;
+ }
+
+ /**
+ * This method will return one row at a time based on the counter given.
+ * @param counter
+ * @return
+ */
+ public Object[] getRawRow(int counter) {
+ return rows.get(counter);
+ }
+
+ /**
+ * For getting the total size.
+ * @return
+ */
+ public int getSize() {
+ return rows.size();
+ }
+
+
+ /**
+ * Returns {@code true} if the iteration has more elements.
+ *
+ * @return {@code true} if the iteration has more elements
+ */
+ @Override public boolean hasNext() {
+ return counter < rows.size();
+ }
+
+ /**
+ * Returns the next element in the iteration.
+ *
+ * @return the next element in the iteration
+ */
+ @Override public Object[] next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ Object[] row = rows.get(counter);
+ counter++;
+ return row;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
index 8120310..bcc5634 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
@@ -17,7 +17,7 @@
package org.apache.carbondata.core.scan.result.impl;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
/**
@@ -25,7 +25,7 @@ import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
* In case of filter query data will be send
* based on filtered row index
*/
-public class FilterQueryScannedResult extends AbstractScannedResult {
+public class FilterQueryScannedResult extends BlockletScannedResult {
public FilterQueryScannedResult(BlockExecutionInfo tableBlockExecutionInfos) {
super(tableBlockExecutionInfos);
@@ -37,7 +37,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
@Override public byte[] getDictionaryKeyArray() {
++currentRow;
- return getDictionaryKeyArray(rowMapping[pageCounter][currentRow]);
+ return getDictionaryKeyArray(pageFilteredRowId[pageCounter][currentRow]);
}
/**
@@ -46,7 +46,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
@Override public int[] getDictionaryKeyIntegerArray() {
++currentRow;
- return getDictionaryKeyIntegerArray(rowMapping[pageCounter][currentRow]);
+ return getDictionaryKeyIntegerArray(pageFilteredRowId[pageCounter][currentRow]);
}
/**
@@ -55,7 +55,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
* @return complex type key array
*/
@Override public byte[][] getComplexTypeKeyArray() {
- return getComplexTypeKeyArray(rowMapping[pageCounter][currentRow]);
+ return getComplexTypeKeyArray(pageFilteredRowId[pageCounter][currentRow]);
}
/**
@@ -65,17 +65,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
* @return no dictionary key array for all the no dictionary dimension
*/
@Override public byte[][] getNoDictionaryKeyArray() {
- return getNoDictionaryKeyArray(rowMapping[pageCounter][currentRow]);
- }
-
- /**
- * Below method will be used to get the no dictionary key
- * string array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- @Override public String[] getNoDictionaryKeyStringArray() {
- return getNoDictionaryKeyStringArray(rowMapping[pageCounter][currentRow]);
+ return getNoDictionaryKeyArray(pageFilteredRowId[pageCounter][currentRow]);
}
/**
@@ -84,7 +74,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
* @return valid row id
*/
@Override public int getCurrentRowId() {
- return rowMapping[pageCounter][currentRow];
+ return pageFilteredRowId[pageCounter][currentRow];
}
/**
@@ -92,10 +82,12 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
int column = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[dictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(rowMapping[pageCounter], vectorInfo, column,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ for (int chunkIndex : this.dictionaryColumnChunkIndexes) {
+ column = dimensionColumnPages[chunkIndex][pageCounter].fillVector(
+ pageFilteredRowId[pageCounter],
+ vectorInfo,
+ column,
+ columnGroupKeyStructureInfo.get(chunkIndex));
}
}
@@ -104,10 +96,12 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
int column = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(rowMapping[pageCounter], vectorInfo, column,
- columnGroupKeyStructureInfo.get(noDictionaryColumnBlockIndexes[i]));
+ for (int chunkIndex : this.noDictionaryColumnChunkIndexes) {
+ column = dimensionColumnPages[chunkIndex][pageCounter].fillVector(
+ pageFilteredRowId[pageCounter],
+ vectorInfo,
+ column,
+ columnGroupKeyStructureInfo.get(chunkIndex));
}
}
@@ -116,8 +110,10 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
for (int i = 0; i < measuresOrdinal.length; i++) {
- vectorInfo[i].measureVectorFiller.fillMeasureVectorForFilter(rowMapping[pageCounter],
- measureDataChunks[measuresOrdinal[i]][pageCounter], vectorInfo[i]);
+ vectorInfo[i].measureVectorFiller.fillMeasureVector(
+ pageFilteredRowId[pageCounter],
+ measureColumnPages[measuresOrdinal[i]][pageCounter],
+ vectorInfo[i]);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
index 3978f9e..06687c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
@@ -17,14 +17,14 @@
package org.apache.carbondata.core.scan.result.impl;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
/**
* Result provide class for non filter query
* In case of no filter query we need to return
* complete data
*/
-public class NonFilterQueryScannedResult extends AbstractScannedResult {
+public class NonFilterQueryScannedResult extends BlockletScannedResult {
public NonFilterQueryScannedResult(BlockExecutionInfo blockExecutionInfo) {
super(blockExecutionInfo);
@@ -68,16 +68,6 @@ public class NonFilterQueryScannedResult extends AbstractScannedResult {
}
/**
- * Below method will be used to get the no dictionary key
- * string array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- @Override public String[] getNoDictionaryKeyStringArray() {
- return getNoDictionaryKeyStringArray(currentRow);
- }
-
- /**
* will return the current valid row id
*
* @return valid row id
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 6172b40..4e628fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -28,18 +28,17 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.DataRefNodeFinder;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNodeWrapper;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNode;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.DeleteDeltaInfo;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
-import org.apache.carbondata.core.scan.processor.impl.DataBlockIteratorImpl;
+import org.apache.carbondata.core.scan.processor.DataBlockIterator;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
@@ -63,23 +62,23 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
private static final Map<DeleteDeltaInfo, Object> deleteDeltaToLockObjectMap =
new ConcurrentHashMap<>();
- protected ExecutorService execService;
+ private ExecutorService execService;
/**
* execution info of the block
*/
- protected List<BlockExecutionInfo> blockExecutionInfos;
+ private List<BlockExecutionInfo> blockExecutionInfos;
/**
* file reader which will be used to execute the query
*/
- protected FileHolder fileReader;
+ protected FileReader fileReader;
- protected AbstractDataBlockIterator dataBlockIterator;
+ DataBlockIterator dataBlockIterator;
/**
* QueryStatisticsRecorder
*/
- protected QueryStatisticsRecorder recorder;
+ private QueryStatisticsRecorder recorder;
/**
* number of cores which can be used
*/
@@ -89,7 +88,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
*/
private QueryStatisticsModel queryStatisticsModel;
- public AbstractDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
+ AbstractDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
ExecutorService execService) {
String batchSizeString =
CarbonProperties.getInstance().getProperty(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE);
@@ -107,7 +106,6 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
this.blockExecutionInfos = infos;
this.fileReader = FileFactory.getFileHolder(
FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getTablePath()));
- this.fileReader.setQueryId(queryModel.getQueryId());
this.fileReader.setReadPageByPage(queryModel.isReadPageByPage());
this.execService = execService;
intialiseInfos();
@@ -130,22 +128,21 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
blockInfo.setDeletedRecordsMap(deletedRowsMap);
}
DataRefNode dataRefNode = blockInfo.getDataBlock().getDataRefNode();
- if (dataRefNode instanceof BlockletDataRefNodeWrapper) {
- BlockletDataRefNodeWrapper wrapper = (BlockletDataRefNodeWrapper) dataRefNode;
- blockInfo.setFirstDataBlock(wrapper);
- blockInfo.setNumberOfBlockToScan(wrapper.numberOfNodes());
-
+ if (dataRefNode instanceof BlockletDataRefNode) {
+ BlockletDataRefNode node = (BlockletDataRefNode) dataRefNode;
+ blockInfo.setFirstDataBlock(node);
+ blockInfo.setNumberOfBlockToScan(node.numberOfNodes());
} else {
DataRefNode startDataBlock =
finder.findFirstDataBlock(dataRefNode, blockInfo.getStartKey());
- while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
+ while (startDataBlock.nodeIndex() < blockInfo.getStartBlockletIndex()) {
startDataBlock = startDataBlock.getNextDataRefNode();
}
long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan();
//if number of block is less than 0 then take end block.
if (numberOfBlockToScan <= 0) {
DataRefNode endDataBlock = finder.findLastDataBlock(dataRefNode, blockInfo.getEndKey());
- numberOfBlockToScan = endDataBlock.nodeNumber() - startDataBlock.nodeNumber() + 1;
+ numberOfBlockToScan = endDataBlock.nodeIndex() - startDataBlock.nodeIndex() + 1;
}
blockInfo.setFirstDataBlock(startDataBlock);
blockInfo.setNumberOfBlockToScan(numberOfBlockToScan);
@@ -230,7 +227,8 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
}
}
- @Override public boolean hasNext() {
+ @Override
+ public boolean hasNext() {
if ((dataBlockIterator != null && dataBlockIterator.hasNext())) {
return true;
} else if (blockExecutionInfos.size() > 0) {
@@ -240,7 +238,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
}
}
- protected void updateDataBlockIterator() {
+ void updateDataBlockIterator() {
if (dataBlockIterator == null || !dataBlockIterator.hasNext()) {
dataBlockIterator = getDataBlockIterator();
while (dataBlockIterator != null && !dataBlockIterator.hasNext()) {
@@ -249,17 +247,17 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
}
}
- private DataBlockIteratorImpl getDataBlockIterator() {
+ private DataBlockIterator getDataBlockIterator() {
if (blockExecutionInfos.size() > 0) {
BlockExecutionInfo executionInfo = blockExecutionInfos.get(0);
blockExecutionInfos.remove(executionInfo);
- return new DataBlockIteratorImpl(executionInfo, fileReader, batchSize, queryStatisticsModel,
+ return new DataBlockIterator(executionInfo, fileReader, batchSize, queryStatisticsModel,
execService);
}
return null;
}
- protected void initQueryStatiticsModel() {
+ private void initQueryStatiticsModel() {
this.queryStatisticsModel = new QueryStatisticsModel();
this.queryStatisticsModel.setRecorder(recorder);
QueryStatistic queryStatisticTotalBlocklet = new QueryStatistic();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
index 1efac30..1235789 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
@@ -18,7 +18,7 @@
package org.apache.carbondata.core.scan.result.iterator;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
/**
* Iterator over row result
@@ -28,14 +28,14 @@ public class ChunkRowIterator extends CarbonIterator<Object[]> {
/**
* iterator over chunk result
*/
- private CarbonIterator<BatchResult> iterator;
+ private CarbonIterator<RowBatch> iterator;
/**
* currect chunk
*/
- private BatchResult currentchunk;
+ private RowBatch currentchunk;
- public ChunkRowIterator(CarbonIterator<BatchResult> iterator) {
+ public ChunkRowIterator(CarbonIterator<RowBatch> iterator) {
this.iterator = iterator;
if (iterator.hasNext()) {
currentchunk = iterator.next();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
index 747f5a9..c073c78 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
@@ -21,14 +21,14 @@ import java.util.concurrent.ExecutorService;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
/**
* In case of detail query we cannot keep all the records in memory so for
* executing that query are returning a iterator over block and every time next
* call will come it will execute the block and return the result
*/
-public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator<BatchResult> {
+public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator<RowBatch> {
private final Object lock = new Object();
@@ -37,18 +37,18 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
super(infos, queryModel, execService);
}
- @Override public BatchResult next() {
+ @Override public RowBatch next() {
return getBatchResult();
}
- private BatchResult getBatchResult() {
- BatchResult batchResult = new BatchResult();
+ private RowBatch getBatchResult() {
+ RowBatch rowBatch = new RowBatch();
synchronized (lock) {
updateDataBlockIterator();
if (dataBlockIterator != null) {
- batchResult.setRows(dataBlockIterator.next());
+ rowBatch.setRows(dataBlockIterator.next());
}
}
- return batchResult;
+ return rowBatch;
}
}
[30/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 5d927df..73da878 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -35,8 +35,8 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
@@ -100,7 +100,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
/**
* Implementation of RecordReader API.
*/
- @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
+ @Override
+ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
throws IOException, InterruptedException, UnsupportedOperationException {
// The input split can contain single HDFS block or multiple blocks, so firstly get all the
// blocks and then set them in the query model.
@@ -145,7 +146,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
}
}
- @Override public void close() throws IOException {
+ @Override
+ public void close() throws IOException {
logStatistics(rowCount, queryModel.getStatisticsRecorder());
if (columnarBatch != null) {
columnarBatch.close();
@@ -165,10 +167,13 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
}
}
- @Override public boolean nextKeyValue() throws IOException, InterruptedException {
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
resultBatch();
- if (returnColumnarBatch) return nextBatch();
+ if (returnColumnarBatch) {
+ return nextBatch();
+ }
if (batchIdx >= numBatched) {
if (!nextBatch()) return false;
@@ -177,7 +182,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
return true;
}
- @Override public Object getCurrentValue() throws IOException, InterruptedException {
+ @Override
+ public Object getCurrentValue() throws IOException, InterruptedException {
if (returnColumnarBatch) {
int value = columnarBatch.numValidRows();
rowCount += value;
@@ -190,11 +196,13 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
return columnarBatch.getRow(batchIdx - 1);
}
- @Override public Void getCurrentKey() throws IOException, InterruptedException {
+ @Override
+ public Void getCurrentKey() throws IOException, InterruptedException {
return null;
}
- @Override public float getProgress() throws IOException, InterruptedException {
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
// TODO : Implement it based on total number of rows it is going to retrive.
return 0;
}
@@ -206,44 +214,44 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
*/
private void initBatch(MemoryMode memMode) {
- List<QueryDimension> queryDimension = queryModel.getQueryDimension();
- List<QueryMeasure> queryMeasures = queryModel.getQueryMeasures();
+ List<ProjectionDimension> queryDimension = queryModel.getProjectionDimensions();
+ List<ProjectionMeasure> queryMeasures = queryModel.getProjectionMeasures();
StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
for (int i = 0; i < queryDimension.size(); i++) {
- QueryDimension dim = queryDimension.get(i);
+ ProjectionDimension dim = queryDimension.get(i);
if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dim.getDimension().getDataType());
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(generator.getReturnType()), true, null);
} else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true,
null);
} else if (dim.getDimension().isComplex()) {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true,
null);
} else {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(DataTypes.INT), true, null);
}
}
for (int i = 0; i < queryMeasures.size(); i++) {
- QueryMeasure msr = queryMeasures.get(i);
+ ProjectionMeasure msr = queryMeasures.get(i);
DataType dataType = msr.getMeasure().getDataType();
if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT ||
dataType == DataTypes.INT || dataType == DataTypes.LONG) {
- fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
+ fields[msr.getOrdinal()] = new StructField(msr.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true,
null);
} else if (DataTypes.isDecimal(dataType)) {
- fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
+ fields[msr.getOrdinal()] = new StructField(msr.getColumnName(),
new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale()), true,
null);
} else {
- fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
+ fields[msr.getOrdinal()] = new StructField(msr.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(DataTypes.DOUBLE), true, null);
}
}
@@ -261,9 +269,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
initBatch(DEFAULT_MEMORY_MODE);
}
- private ColumnarBatch resultBatch() {
+ private void resultBatch() {
if (columnarBatch == null) initBatch();
- return columnarBatch;
}
/*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
index f51ced3..6a401d8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
@@ -34,20 +34,16 @@ import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.DataTypeConverter;
/**
* Executor class for executing the query on the selected segments to be merged.
@@ -70,6 +66,9 @@ public class CarbonCompactionExecutor {
*/
private boolean restructuredBlockExists;
+ // converter for UTF8String and decimal conversion
+ private DataTypeConverter dataTypeConverter;
+
/**
* Constructor
*
@@ -82,13 +81,14 @@ public class CarbonCompactionExecutor {
public CarbonCompactionExecutor(Map<String, TaskBlockInfo> segmentMapping,
SegmentProperties segmentProperties, CarbonTable carbonTable,
Map<String, List<DataFileFooter>> dataFileMetadataSegMapping,
- boolean restructuredBlockExists) {
+ boolean restructuredBlockExists, DataTypeConverter dataTypeConverter) {
this.segmentMapping = segmentMapping;
this.destinationSegProperties = segmentProperties;
this.carbonTable = carbonTable;
this.dataFileMetadataSegMapping = dataFileMetadataSegMapping;
this.restructuredBlockExists = restructuredBlockExists;
- queryExecutorList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ this.queryExecutorList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ this.dataTypeConverter = dataTypeConverter;
}
/**
@@ -100,7 +100,9 @@ public class CarbonCompactionExecutor {
List<RawResultIterator> resultList =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<TableBlockInfo> list = null;
- queryModel = prepareQueryModel(list);
+ queryModel = carbonTable.createQueryModelWithProjectAllColumns(dataTypeConverter);
+ queryModel.setReadPageByPage(enablePageLevelReaderForCompaction());
+ queryModel.setForcedDetailRawQuery(true);
// iterate each seg ID
for (Map.Entry<String, TaskBlockInfo> taskMap : segmentMapping.entrySet()) {
String segmentId = taskMap.getKey();
@@ -156,7 +158,7 @@ public class CarbonCompactionExecutor {
* @param blockList
* @return
*/
- private CarbonIterator<BatchResult> executeBlockList(List<TableBlockInfo> blockList)
+ private CarbonIterator<RowBatch> executeBlockList(List<TableBlockInfo> blockList)
throws QueryExecutionException, IOException {
queryModel.setTableBlockInfos(blockList);
QueryExecutor queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
@@ -195,48 +197,6 @@ public class CarbonCompactionExecutor {
}
/**
- * Preparing of the query model.
- *
- * @param blockList
- * @return
- */
- private QueryModel prepareQueryModel(List<TableBlockInfo> blockList) {
- QueryModel model = new QueryModel();
- model.setTableBlockInfos(blockList);
- model.setForcedDetailRawQuery(true);
- model.setFilterExpressionResolverTree(null);
- model.setConverter(DataTypeUtil.getDataTypeConverter());
- model.setReadPageByPage(enablePageLevelReaderForCompaction());
-
- List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getTableName());
- for (CarbonDimension dim : dimensions) {
- // check if dimension is deleted
- QueryDimension queryDimension = new QueryDimension(dim.getColName());
- queryDimension.setDimension(dim);
- dims.add(queryDimension);
- }
- model.setQueryDimension(dims);
-
- List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- List<CarbonMeasure> measures =
- carbonTable.getMeasureByTableName(carbonTable.getTableName());
- for (CarbonMeasure carbonMeasure : measures) {
- // check if measure is deleted
- QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
- queryMeasure.setMeasure(carbonMeasure);
- msrs.add(queryMeasure);
- }
- model.setQueryMeasures(msrs);
- model.setQueryId(System.nanoTime() + "");
- model.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
- model.setTable(carbonTable);
- return model;
- }
-
- /**
* Whether to enable page level reader for compaction or not.
*/
private boolean enablePageLevelReaderForCompaction() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java b/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java
index 79e9e5a..b6f12a5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.processing.partition.DataPartitioner;
import org.apache.carbondata.processing.partition.Partition;
@@ -46,9 +45,8 @@ public final class QueryPartitionHelper {
/**
* Get partitions applicable for query based on filters applied in query
*/
- public List<Partition> getPartitionsForQuery(CarbonQueryPlan queryPlan) {
- String tableUniqueName =
- CarbonTable.buildUniqueName(queryPlan.getDatabaseName(), queryPlan.getTableName());
+ public List<Partition> getPartitionsForQuery(String databaseName, String tableName) {
+ String tableUniqueName = CarbonTable.buildUniqueName(databaseName, tableName);
DataPartitioner dataPartitioner = partitionerMap.get(tableUniqueName);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
index 36e022b..01db4f6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
@@ -18,7 +18,6 @@
package org.apache.carbondata.processing.partition.spliter;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -26,19 +25,14 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.util.CarbonUtil;
public abstract class AbstractCarbonQueryExecutor {
@@ -47,8 +41,8 @@ public abstract class AbstractCarbonQueryExecutor {
LogServiceFactory.getLogService(AbstractCarbonQueryExecutor.class.getName());
protected CarbonTable carbonTable;
protected QueryModel queryModel;
- protected QueryExecutor queryExecutor;
- protected Map<String, TaskBlockInfo> segmentMapping;
+ private QueryExecutor queryExecutor;
+ Map<String, TaskBlockInfo> segmentMapping;
/**
* get executor and execute the query model.
@@ -56,7 +50,7 @@ public abstract class AbstractCarbonQueryExecutor {
* @param blockList
* @return
*/
- protected CarbonIterator<BatchResult> executeBlockList(List<TableBlockInfo> blockList)
+ CarbonIterator<RowBatch> executeBlockList(List<TableBlockInfo> blockList)
throws QueryExecutionException, IOException {
queryModel.setTableBlockInfos(blockList);
this.queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
@@ -64,46 +58,6 @@ public abstract class AbstractCarbonQueryExecutor {
}
/**
- * Preparing of the query model.
- *
- * @param blockList
- * @return
- */
- protected QueryModel prepareQueryModel(List<TableBlockInfo> blockList) {
- QueryModel model = new QueryModel();
- model.setTableBlockInfos(blockList);
- model.setForcedDetailRawQuery(true);
- model.setFilterExpressionResolverTree(null);
-
- List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getTableName());
- for (CarbonDimension dim : dimensions) {
- // check if dimension is deleted
- QueryDimension queryDimension = new QueryDimension(dim.getColName());
- queryDimension.setDimension(dim);
- dims.add(queryDimension);
- }
- model.setQueryDimension(dims);
-
- List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- List<CarbonMeasure> measures =
- carbonTable.getMeasureByTableName(carbonTable.getTableName());
- for (CarbonMeasure carbonMeasure : measures) {
- // check if measure is deleted
- QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
- queryMeasure.setMeasure(carbonMeasure);
- msrs.add(queryMeasure);
- }
- model.setQueryMeasures(msrs);
- model.setQueryId(System.nanoTime() + "");
- model.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
- model.setTable(carbonTable);
- return model;
- }
-
- /**
* Below method will be used
* for cleanup
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java
index 6afec0b..b18207d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.core.scan.result.iterator.PartitionSpliterRawResultIterator;
+import org.apache.carbondata.core.util.DataTypeConverterImpl;
/**
* Used to read carbon blocks when add/split partition
@@ -48,7 +49,8 @@ public class CarbonSplitExecutor extends AbstractCarbonQueryExecutor {
public List<PartitionSpliterRawResultIterator> processDataBlocks(String segmentId)
throws QueryExecutionException, IOException {
List<TableBlockInfo> list = null;
- queryModel = prepareQueryModel(list);
+ queryModel = carbonTable.createQueryModelWithProjectAllColumns(new DataTypeConverterImpl());
+ queryModel.setForcedDetailRawQuery(true);
List<PartitionSpliterRawResultIterator> resultList
= new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
TaskBlockInfo taskBlockInfo = segmentMapping.get(segmentId);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java
index ec91472..4abdf3c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java
@@ -24,7 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
+import org.apache.carbondata.core.scan.model.QueryProjection;
import org.apache.carbondata.processing.partition.Partition;
import org.apache.carbondata.processing.partition.impl.DefaultLoadBalancer;
import org.apache.carbondata.processing.partition.impl.PartitionMultiFileImpl;
@@ -46,7 +46,7 @@ public class CarbonQueryUtil {
* It creates the one split for each region server.
*/
public static synchronized TableSplit[] getTableSplits(String databaseName, String tableName,
- CarbonQueryPlan queryPlan) {
+ QueryProjection queryPlan) {
//Just create splits depends on locations of region servers
List<Partition> allPartitions = null;
@@ -55,7 +55,7 @@ public class CarbonQueryUtil {
QueryPartitionHelper.getInstance().getAllPartitions(databaseName, tableName);
} else {
allPartitions =
- QueryPartitionHelper.getInstance().getPartitionsForQuery(queryPlan);
+ QueryPartitionHelper.getInstance().getPartitionsForQuery(databaseName, tableName);
}
TableSplit[] splits = new TableSplit[allPartitions.size()];
for (int i = 0; i < splits.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 36a5a15..197cb14 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -150,7 +150,7 @@ class StreamHandoffRDD[K, V](
CarbonTableInputFormat.setTableInfo(hadoopConf, carbonTable.getTableInfo)
val attemptContext = new TaskAttemptContextImpl(hadoopConf, attemptId)
val format = new CarbonTableInputFormat[Array[Object]]()
- val model = format.getQueryModel(inputSplit, attemptContext)
+ val model = format.createQueryModel(inputSplit, attemptContext)
val inputFormat = new CarbonStreamInputFormat
val streamReader = inputFormat.createRecordReader(inputSplit, attemptContext)
.asInstanceOf[CarbonStreamRecordReader]
[18/50] [abbrv] carbondata git commit: [CARBONDATA-2044]
Documentation added 2 parameter
Posted by ra...@apache.org.
[CARBONDATA-2044] Documentation added 2 parameter
Documentation added 2 parameter
This closes #1893
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8a86d3fd
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8a86d3fd
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8a86d3fd
Branch: refs/heads/fgdatamap
Commit: 8a86d3fd081bd78f3de551d5b7b696bdcf517ed5
Parents: 15fabf9
Author: sgururajshetty <sg...@gmail.com>
Authored: Wed Jan 31 16:24:13 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jan 31 19:15:01 2018 +0800
----------------------------------------------------------------------
docs/configuration-parameters.md | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8a86d3fd/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index cfe24a5..522d222 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -127,7 +127,9 @@ This section provides the details of all the configurations required for CarbonD
| Parameter | Default Value | Description |
|--------------------------|---------------|-----------------------------------------------------------------------------------------------|
| max.query.execution.time | 60 | Maximum time allowed for one query to be executed. The value is in minutes. |
-| carbon.enableMinMax | true | Min max is feature added to enhance query performance. To disable this feature, set it false. |
+| carbon.enableMinMax | true | Min max is feature added to enhance query performance. To disable this feature, set it false. |
+| carbon.dynamicallocation.schedulertimeout | 5 | Specifies the maximum time (unit in seconds) the scheduler can wait for executor to be active. Minimum value is 5 sec and maximum value is 15 sec. |
+| carbon.scheduler.minregisteredresourcesratio | 0.8 | Specifies the minimum resource (executor) ratio needed for starting the block distribution. The default value is 0.8, which indicates 80% of the requested resource is allocated for starting block distribution. The minimum value is 0.1 min and the maximum value is 1.0. |
* **Global Dictionary Configurations**
[46/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all
path construction through CarbonTablePath static method
Posted by ra...@apache.org.
[CARBONDATA-2025] Unify all path construction through CarbonTablePath static method
Refactory CarbonTablePath:
1.Remove CarbonStorePath and use CarbonTablePath only.
2.Make CarbonTablePath an utility without object creation, it can avoid creating object before using it, thus code is cleaner and GC is less.
This closes #1768
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c3e99681
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c3e99681
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c3e99681
Branch: refs/heads/fgdatamap
Commit: c3e99681bcd397ed33bc90e8d73b1fd33e0e60f7
Parents: 15b4e19
Author: Jacky Li <ja...@qq.com>
Authored: Wed Jan 31 16:14:27 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Feb 1 15:33:19 2018 +0800
----------------------------------------------------------------------
.../DictionaryColumnUniqueIdentifier.java | 29 +-
.../dictionary/ManageDictionaryAndBTree.java | 13 +-
.../core/metadata/AbsoluteTableIdentifier.java | 4 +-
.../core/metadata/PartitionMapFileStore.java | 15 +-
.../core/metadata/schema/table/CarbonTable.java | 11 +-
.../core/mutate/CarbonUpdateUtil.java | 59 ++--
.../executor/impl/AbstractQueryExecutor.java | 8 +-
.../core/scan/executor/util/QueryUtil.java | 7 +-
.../core/service/CarbonCommonFactory.java | 16 -
.../carbondata/core/service/PathService.java | 35 ---
.../core/service/impl/PathFactory.java | 50 ----
.../statusmanager/SegmentStatusManager.java | 28 +-
.../SegmentUpdateStatusManager.java | 72 ++---
.../apache/carbondata/core/util/CarbonUtil.java | 61 ++--
.../util/path/CarbonSharedDictionaryPath.java | 71 -----
.../core/util/path/CarbonStorePath.java | 71 -----
.../core/util/path/CarbonTablePath.java | 296 ++++++-------------
.../dictionary/AbstractDictionaryCacheTest.java | 11 +-
.../dictionary/ForwardDictionaryCacheTest.java | 6 +-
.../dictionary/ReverseDictionaryCacheTest.java | 6 +-
.../reader/CarbonDictionaryReaderImplTest.java | 8 -
.../CarbonFormatDirectoryStructureTest.java | 18 +-
.../path/CarbonFormatSharedDictionaryTest.java | 44 ---
.../writer/CarbonDictionaryWriterImplTest.java | 19 +-
.../CarbonBatchSparkStreamingExample.scala | 9 +-
.../CarbonStructuredStreamingExample.scala | 11 +-
.../hadoop/api/CarbonTableInputFormat.java | 8 +-
.../streaming/CarbonStreamRecordWriter.java | 6 +-
.../carbondata/hadoop/util/SchemaReader.java | 18 +-
.../hadoop/test/util/StoreCreator.java | 16 +-
.../presto/CarbondataRecordSetProvider.java | 7 +-
.../presto/impl/CarbonTableCacheModel.java | 13 +-
.../presto/impl/CarbonTableReader.java | 49 ++-
.../presto/util/CarbonDataStoreCreator.scala | 10 +-
.../sdv/generated/MergeIndexTestCase.scala | 2 +-
.../dataload/TestLoadDataGeneral.scala | 11 +-
.../InsertIntoCarbonTableTestCase.scala | 8 +-
.../TestLoadTableConcurrentScenario.scala | 78 -----
.../createTable/TestCreateTableAsSelect.scala | 2 +-
.../datacompaction/DataCompactionLockTest.scala | 6 +-
.../MajorCompactionIgnoreInMinorTest.scala | 12 +-
.../dataload/TestBatchSortDataLoad.scala | 5 +-
.../dataload/TestDataLoadWithFileName.scala | 5 +-
.../dataload/TestGlobalSortDataLoad.scala | 8 +-
.../dataretention/DataRetentionTestCase.scala | 6 +-
.../TestDataLoadingForPartitionTable.scala | 4 +-
.../StandardPartitionTableCleanTestCase.scala | 4 +-
...andardPartitionTableCompactionTestCase.scala | 4 +-
.../StandardPartitionTableLoadingTestCase.scala | 8 +-
.../org/apache/carbondata/api/CarbonStore.scala | 4 +-
.../carbondata/spark/CarbonSparkFactory.scala | 2 +-
.../spark/DictionaryDetailHelper.scala | 9 +-
.../spark/rdd/AlterTableAddColumnRDD.scala | 17 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 6 -
.../carbondata/spark/util/CommonUtil.scala | 30 +-
.../carbondata/spark/util/DataLoadingUtil.scala | 10 +-
.../spark/util/GlobalDictionaryUtil.scala | 16 +-
.../command/carbonTableSchemaCommon.scala | 10 +-
.../org/apache/spark/util/PartitionUtils.scala | 6 +-
.../spark/rdd/AggregateDataMapCompactor.scala | 13 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 14 +-
.../spark/rdd/CarbonTableCompactor.scala | 8 +-
.../CarbonAlterTableCompactionCommand.scala | 14 +-
.../management/CarbonLoadDataCommand.scala | 14 +-
.../management/CarbonShowLoadsCommand.scala | 2 +-
.../management/RefreshCarbonTableCommand.scala | 17 +-
.../command/mutation/DeleteExecution.scala | 5 +-
.../CarbonAlterTableDropPartitionCommand.scala | 18 +-
.../CarbonAlterTableSplitPartitionCommand.scala | 22 +-
.../CreatePreAggregateTableCommand.scala | 2 +-
.../preaaggregate/PreAggregateUtil.scala | 15 +-
.../CarbonAlterTableAddColumnCommand.scala | 8 +-
.../CarbonAlterTableDataTypeChangeCommand.scala | 6 +-
.../CarbonAlterTableDropColumnCommand.scala | 6 +-
.../schema/CarbonAlterTableRenameCommand.scala | 28 +-
.../spark/sql/hive/CarbonFileMetastore.scala | 27 +-
.../spark/sql/hive/CarbonHiveMetaStore.scala | 13 +-
.../apache/spark/sql/hive/CarbonMetaStore.scala | 2 +-
.../apache/spark/sql/hive/CarbonRelation.scala | 6 +-
.../org/apache/spark/util/AlterTableUtil.scala | 54 ++--
.../partition/TestAlterPartitionTable.scala | 4 +-
.../spark/util/AllDictionaryTestCase.scala | 6 +-
.../spark/util/DictionaryTestCaseUtil.scala | 2 +-
.../util/ExternalColumnDictionaryTestCase.scala | 6 +-
.../TestStreamingTableOperation.scala | 24 +-
.../restructure/AlterTableRevertTestCase.scala | 2 +-
.../loading/TableProcessingOperations.java | 59 ++--
.../converter/impl/FieldEncoderFactory.java | 9 +-
.../merger/AbstractResultProcessor.java | 8 +-
.../processing/merger/CarbonCompactionUtil.java | 2 +-
.../processing/merger/CarbonDataMergerUtil.java | 84 ++----
.../merger/CompactionResultSortProcessor.java | 3 +-
.../merger/RowResultMergerProcessor.java | 3 +-
.../store/CarbonFactDataHandlerModel.java | 9 +-
.../util/CarbonDataProcessorUtil.java | 16 +-
.../processing/util/CarbonLoaderUtil.java | 95 +++---
.../processing/util/DeleteLoadFolders.java | 8 +-
.../carbon/datastore/BlockIndexStoreTest.java | 36 +--
.../carbondata/processing/StoreCreator.java | 34 +--
.../streaming/segment/StreamSegment.java | 24 +-
.../carbondata/streaming/StreamHandoffRDD.scala | 22 +-
.../streaming/StreamSinkFactory.scala | 14 +-
.../streaming/CarbonAppendableStreamSink.scala | 18 +-
103 files changed, 674 insertions(+), 1486 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
index f5652ac..a7b47c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
@@ -22,7 +22,6 @@ import java.io.Serializable;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
/**
@@ -42,8 +41,6 @@ public class DictionaryColumnUniqueIdentifier implements Serializable {
*/
private ColumnIdentifier columnIdentifier;
- private transient CarbonTablePath carbonTablePath;
-
private DataType dataType;
private String dictionaryLocation;
@@ -68,8 +65,7 @@ public class DictionaryColumnUniqueIdentifier implements Serializable {
this.columnIdentifier = columnIdentifier;
this.dataType = columnIdentifier.getDataType();
this.dictionaryLocation =
- CarbonStorePath.getCarbonTablePath(dictionarySourceAbsoluteTableIdentifier)
- .getMetadataDirectoryPath();
+ CarbonTablePath.getMetadataPath(dictionarySourceAbsoluteTableIdentifier.getTablePath());
}
/**
@@ -100,13 +96,6 @@ public class DictionaryColumnUniqueIdentifier implements Serializable {
return dataType;
}
- public CarbonTablePath getCarbonTablePath() {
- if (null == carbonTablePath) {
- carbonTablePath = CarbonStorePath.getCarbonTablePath(dictionarySourceAbsoluteTableIdentifier);
- }
- return carbonTablePath;
- }
-
/**
* @return columnIdentifier
*/
@@ -118,24 +107,24 @@ public class DictionaryColumnUniqueIdentifier implements Serializable {
* @return dictionary file path
*/
public String getDictionaryFilePath() {
- return getCarbonTablePath()
- .getDictionaryFilePath(dictionaryLocation, columnIdentifier.getColumnId());
+ return CarbonTablePath.getExternalDictionaryFilePath(
+ dictionaryLocation, columnIdentifier.getColumnId());
}
/**
* @return dictionary metadata file path
*/
public String getDictionaryMetaFilePath() {
- return getCarbonTablePath()
- .getDictionaryMetaFilePath(dictionaryLocation, columnIdentifier.getColumnId());
+ return CarbonTablePath.getExternalDictionaryMetaFilePath(
+ dictionaryLocation, columnIdentifier.getColumnId());
}
/**
* @return sort index file path
*/
public String getSortIndexFilePath() {
- return getCarbonTablePath()
- .getSortIndexFilePath(dictionaryLocation, columnIdentifier.getColumnId());
+ return CarbonTablePath.getExternalSortIndexFilePath(
+ dictionaryLocation, columnIdentifier.getColumnId());
}
/**
@@ -143,8 +132,8 @@ public class DictionaryColumnUniqueIdentifier implements Serializable {
* @return sort index file path with given offset
*/
public String getSortIndexFilePath(long offset) {
- return getCarbonTablePath()
- .getSortIndexFilePath(dictionaryLocation, columnIdentifier.getColumnId(), offset);
+ return CarbonTablePath.getExternalSortIndexFilePath(
+ dictionaryLocation, columnIdentifier.getColumnId(), offset);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
index 4f8f724..a7d6027 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
@@ -36,7 +36,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
/**
@@ -56,13 +55,11 @@ public class ManageDictionaryAndBTree {
* clear the dictionary cache
*
* @param columnSchema
- * @param carbonTableIdentifier
+ * @param identifier
*/
public static void deleteDictionaryFileAndCache(final ColumnSchema columnSchema,
- AbsoluteTableIdentifier carbonTableIdentifier) {
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(carbonTableIdentifier);
- String metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath();
+ AbsoluteTableIdentifier identifier) {
+ String metadataDirectoryPath = CarbonTablePath.getMetadataPath(identifier.getTablePath());
CarbonFile metadataDir = FileFactory
.getCarbonFile(metadataDirectoryPath, FileFactory.getFileType(metadataDirectoryPath));
if (metadataDir.exists()) {
@@ -90,7 +87,7 @@ public class ManageDictionaryAndBTree {
}
}
// remove dictionary cache
- removeDictionaryColumnFromCache(carbonTableIdentifier, columnSchema.getColumnUniqueId());
+ removeDictionaryColumnFromCache(identifier, columnSchema.getColumnUniqueId());
}
/**
@@ -101,7 +98,7 @@ public class ManageDictionaryAndBTree {
public static void clearBTreeAndDictionaryLRUCache(CarbonTable carbonTable) {
// clear Btree cache from LRU cache
LoadMetadataDetails[] loadMetadataDetails =
- SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath());
+ SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath());
String[] segments = new String[loadMetadataDetails.length];
int i = 0;
for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
index 6ef2671..5c2f494 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
@@ -36,7 +36,6 @@ public class AbsoluteTableIdentifier implements Serializable {
*/
private String tablePath;
- private boolean isLocalPath;
/**
* carbon table identifier which will have table name and table database
@@ -47,7 +46,6 @@ public class AbsoluteTableIdentifier implements Serializable {
private AbsoluteTableIdentifier(String tablePath, CarbonTableIdentifier carbonTableIdentifier) {
//TODO this should be moved to common place where path handling will be handled
this.tablePath = FileFactory.getUpdatedFilePath(tablePath);
- isLocalPath = tablePath.startsWith(CarbonCommonConstants.LOCAL_FILE_PREFIX);
this.carbonTableIdentifier = carbonTableIdentifier;
}
@@ -79,7 +77,7 @@ public class AbsoluteTableIdentifier implements Serializable {
}
public String appendWithLocalPrefix(String path) {
- if (isLocalPath) {
+ if (tablePath.startsWith(CarbonCommonConstants.LOCAL_FILE_PREFIX)) {
return CarbonCommonConstants.LOCAL_FILE_PREFIX + path;
} else {
return path;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index 43310fe..2744fcd 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -48,7 +48,6 @@ import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataFileFooterConverter;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter;
@@ -343,13 +342,8 @@ public class PartitionMapFileStore {
CarbonTable table,
List<String> currentPartitions,
boolean forceDelete) throws IOException {
- SegmentStatusManager ssm = new SegmentStatusManager(table.getAbsoluteTableIdentifier());
-
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(table.getAbsoluteTableIdentifier().getTablePath(),
- table.getAbsoluteTableIdentifier().getCarbonTableIdentifier());
-
- LoadMetadataDetails[] details = ssm.readLoadMetadata(table.getMetaDataFilepath());
+ LoadMetadataDetails[] details =
+ SegmentStatusManager.readLoadMetadata(table.getMetadataPath());
// scan through each segment.
List<String> segmentsNeedToBeDeleted = new ArrayList<>();
for (LoadMetadataDetails segment : details) {
@@ -363,7 +357,8 @@ public class PartitionMapFileStore {
List<String> toBeDeletedIndexFiles = new ArrayList<>();
List<String> toBeDeletedDataFiles = new ArrayList<>();
// take the list of files from this segment.
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segment.getLoadName());
+ String segmentPath = CarbonTablePath.getSegmentPath(
+ table.getAbsoluteTableIdentifier().getTablePath(), segment.getLoadName());
String partitionFilePath = getPartitionFilePath(segmentPath);
if (partitionFilePath != null) {
PartitionMapper partitionMapper = readPartitionMap(partitionFilePath);
@@ -436,7 +431,7 @@ public class PartitionMapFileStore {
SegmentStatusManager.updateDeletionStatus(
table.getAbsoluteTableIdentifier(),
segmentsNeedToBeDeleted,
- table.getMetaDataFilepath());
+ table.getMetadataPath());
} catch (Exception e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index d5c2609..c9d6aad 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -424,13 +424,20 @@ public class CarbonTable implements Serializable {
}
/**
- * @return the metaDataFilepath
+ * Return the metadata path of the table
*/
- public String getMetaDataFilepath() {
+ public String getMetadataPath() {
return CarbonTablePath.getMetadataPath(getTablePath());
}
/**
+ * Return the segment path of the specified segmentId
+ */
+ public String getSemgentPath(String segmentId) {
+ return CarbonTablePath.getSegmentPath(getTablePath(), segmentId);
+ }
+
+ /**
* @return storepath
*/
public String getTablePath() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 920155b..a564e42 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -42,10 +42,8 @@ import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
-
/**
* This class contains all update utility methods
*/
@@ -182,20 +180,11 @@ public class CarbonUpdateUtil {
CarbonTable table, String updatedTimeStamp,
boolean isTimestampUpdationRequired,
List<String> segmentsToBeDeleted) {
-
boolean status = false;
-
- String metaDataFilepath = table.getMetaDataFilepath();
-
- AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
-
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
-
- String tableStatusPath = carbonTablePath.getTableStatusFilePath();
-
- SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
+ String metaDataFilepath = table.getMetadataPath();
+ AbsoluteTableIdentifier identifier = table.getAbsoluteTableIdentifier();
+ String tableStatusPath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
+ SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
boolean lockStatus = false;
@@ -207,7 +196,7 @@ public class CarbonUpdateUtil {
+ " for table status updation");
LoadMetadataDetails[] listOfLoadFolderDetailsArray =
- segmentStatusManager.readLoadMetadata(metaDataFilepath);
+ SegmentStatusManager.readLoadMetadata(metaDataFilepath);
for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
@@ -290,12 +279,8 @@ public class CarbonUpdateUtil {
*/
public static void cleanStaleDeltaFiles(CarbonTable table, final String timeStamp) {
- AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
-
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
- String partitionDir = carbonTablePath.getPartitionDir();
+ AbsoluteTableIdentifier identifier = table.getAbsoluteTableIdentifier();
+ String partitionDir = CarbonTablePath.getPartitionDir(identifier.getTablePath());
CarbonFile file =
FileFactory.getCarbonFile(partitionDir, FileFactory.getFileType(partitionDir));
if (!file.exists()) {
@@ -365,8 +350,18 @@ public class CarbonUpdateUtil {
.substring(0, completeBlockName.lastIndexOf(CarbonCommonConstants.HYPHEN));
}
- public static long getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
- String segmentDirPath = tablePath.getCarbonDataDirectoryPath(segmentId);
+ /**
+ * returns segment id from segment name
+ *
+ * @param segmentName
+ * @return
+ */
+ public static String getSegmentId(String segmentName) {
+ return segmentName.split(CarbonCommonConstants.UNDERSCORE)[1];
+ }
+
+ public static long getLatestTaskIdForSegment(String segmentId, String tablePath) {
+ String segmentDirPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
// scan all the carbondata files and get the latest task ID.
CarbonFile segment =
@@ -405,11 +400,8 @@ public class CarbonUpdateUtil {
SegmentStatusManager ssm = new SegmentStatusManager(table.getAbsoluteTableIdentifier());
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(table.getAbsoluteTableIdentifier().getTablePath(),
- table.getAbsoluteTableIdentifier().getCarbonTableIdentifier());
-
- LoadMetadataDetails[] details = ssm.readLoadMetadata(table.getMetaDataFilepath());
+ LoadMetadataDetails[] details =
+ SegmentStatusManager.readLoadMetadata(table.getMetadataPath());
String validUpdateStatusFile = "";
@@ -431,7 +423,8 @@ public class CarbonUpdateUtil {
|| segment.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
// take the list of files from this segment.
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segment.getLoadName());
+ String segmentPath = CarbonTablePath.getSegmentPath(
+ table.getAbsoluteTableIdentifier().getTablePath(), segment.getLoadName());
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
@@ -538,8 +531,10 @@ public class CarbonUpdateUtil {
final String updateStatusTimestamp = validUpdateStatusFile
.substring(validUpdateStatusFile.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1);
- CarbonFile metaFolder = FileFactory.getCarbonFile(carbonTablePath.getMetadataDirectoryPath(),
- FileFactory.getFileType(carbonTablePath.getMetadataDirectoryPath()));
+ String tablePath = table.getAbsoluteTableIdentifier().getTablePath();
+ CarbonFile metaFolder = FileFactory.getCarbonFile(
+ CarbonTablePath.getMetadataPath(tablePath),
+ FileFactory.getFileType(CarbonTablePath.getMetadataPath(tablePath)));
CarbonFile[] invalidUpdateStatusFiles = metaFolder.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile file) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 1e73dbb..92c46dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -73,7 +73,7 @@ import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.commons.lang3.ArrayUtils;
@@ -292,10 +292,8 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo,
queryModel.getProjectionDimensions(), tableBlockDimensions,
segmentProperties.getComplexDimensions());
- int tableFactPathLength = CarbonStorePath
- .getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getTablePath(),
- queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier()).getFactDir()
- .length() + 1;
+ int tableFactPathLength = CarbonTablePath.getFactDir(
+ queryModel.getAbsoluteTableIdentifier().getTablePath()).length() + 1;
blockExecutionInfo.setBlockId(filePath.substring(tableFactPathLength));
blockExecutionInfo.setDeleteDeltaFilePath(deleteDeltaFiles);
blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 00cdfe7..1765efa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -63,7 +63,6 @@ import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnRes
import org.apache.carbondata.core.scan.model.ProjectionDimension;
import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.commons.lang3.ArrayUtils;
@@ -409,7 +408,7 @@ public class QueryUtil {
}
public static AbsoluteTableIdentifier getTableIdentifierForColumn(CarbonDimension carbonDimension,
- AbsoluteTableIdentifier absoluteTableIdentifier) {
+ AbsoluteTableIdentifier identifier) {
RelationIdentifier relation = carbonDimension.getColumnSchema()
.getParentColumnTableRelations()
.get(0)
@@ -417,8 +416,8 @@ public class QueryUtil {
String parentTableName = relation.getTableName();
String parentDatabaseName = relation.getDatabaseName();
String parentTableId = relation.getTableId();
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- String newTablePath = CarbonUtil.getNewTablePath(carbonTablePath, parentTableName);
+ String newTablePath =
+ CarbonTablePath.getNewTablePath(identifier.getTablePath(), parentTableName);
return AbsoluteTableIdentifier.from(newTablePath, parentDatabaseName, parentTableName,
parentTableId);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java b/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
index 6fd9735..3ddf00b 100644
--- a/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
@@ -16,9 +16,7 @@
*/
package org.apache.carbondata.core.service;
-import org.apache.carbondata.core.service.impl.ColumnUniqueIdGenerator;
import org.apache.carbondata.core.service.impl.DictionaryFactory;
-import org.apache.carbondata.core.service.impl.PathFactory;
/**
* Interface to get services
@@ -32,18 +30,4 @@ public class CarbonCommonFactory {
return DictionaryFactory.getInstance();
}
- /**
- * @return path service
- */
- public static PathService getPathService() {
- return PathFactory.getInstance();
- }
-
- /**
- * @return unique id generator
- */
- public static ColumnUniqueIdService getColumnUniqueIdGenerator() {
- return ColumnUniqueIdGenerator.getInstance();
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/service/PathService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/PathService.java b/core/src/main/java/org/apache/carbondata/core/service/PathService.java
deleted file mode 100644
index 664d045..0000000
--- a/core/src/main/java/org/apache/carbondata/core/service/PathService.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.service;
-
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-
-/**
- * Create helper to get path details
- */
-public interface PathService {
-
- /**
- * @param absoluteTableIdentifier
- * @param dictionaryColumnUniqueIdentifier
- * @return store path related to tables
- */
- CarbonTablePath getCarbonTablePath(AbsoluteTableIdentifier absoluteTableIdentifier,
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier);
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
deleted file mode 100644
index cb4ca25..0000000
--- a/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.service.impl;
-
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.service.PathService;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-
-/**
- * Create helper to get path details
- */
-public class PathFactory implements PathService {
-
- private static PathService pathService = new PathFactory();
-
- /**
- * @param absoluteTableIdentifier
- * @param
- * @return store path related to tables
- */
- @Override public CarbonTablePath getCarbonTablePath(
- AbsoluteTableIdentifier absoluteTableIdentifier,
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
- if (null != dictionaryColumnUniqueIdentifier && null != dictionaryColumnUniqueIdentifier
- .getCarbonTablePath()) {
- return dictionaryColumnUniqueIdentifier.getCarbonTablePath();
- }
- return CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- }
-
- public static PathService getInstance() {
- return pathService;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index e02f246..c613735 100755
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -46,7 +46,6 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import com.google.gson.Gson;
@@ -59,10 +58,10 @@ public class SegmentStatusManager {
private static final LogService LOG =
LogServiceFactory.getLogService(SegmentStatusManager.class.getName());
- private AbsoluteTableIdentifier absoluteTableIdentifier;
+ private AbsoluteTableIdentifier identifier;
- public SegmentStatusManager(AbsoluteTableIdentifier absoluteTableIdentifier) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
+ public SegmentStatusManager(AbsoluteTableIdentifier identifier) {
+ this.identifier = identifier;
}
/**
@@ -71,7 +70,7 @@ public class SegmentStatusManager {
* @return
*/
public ICarbonLock getTableStatusLock() {
- return CarbonLockFactory.getCarbonLockObj(absoluteTableIdentifier, LockUsage.TABLE_STATUS_LOCK);
+ return CarbonLockFactory.getCarbonLockObj(identifier, LockUsage.TABLE_STATUS_LOCK);
}
/**
@@ -79,9 +78,7 @@ public class SegmentStatusManager {
*/
public static long getTableStatusLastModifiedTime(AbsoluteTableIdentifier identifier)
throws IOException {
- String tableStatusPath = CarbonStorePath
- .getCarbonTablePath(identifier.getTablePath(), identifier.getCarbonTableIdentifier())
- .getTableStatusFilePath();
+ String tableStatusPath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
if (!FileFactory.isFileExist(tableStatusPath, FileFactory.getFileType(tableStatusPath))) {
return 0L;
} else {
@@ -103,10 +100,7 @@ public class SegmentStatusManager {
List<String> listOfValidUpdatedSegments = new ArrayList<>(10);
List<String> listOfInvalidSegments = new ArrayList<>(10);
List<String> listOfStreamSegments = new ArrayList<>(10);
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
- String dataPath = carbonTablePath.getTableStatusFilePath();
+ String dataPath = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
DataInputStream dataInputStream = null;
// Use GSON to deserialize the load information
@@ -280,9 +274,7 @@ public class SegmentStatusManager {
if (carbonDeleteSegmentLock.lockWithRetries()) {
LOG.info("Delete segment lock has been successfully acquired");
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(
- identifier.getTablePath(), identifier.getCarbonTableIdentifier());
- String dataLoadLocation = carbonTablePath.getTableStatusFilePath();
+ String dataLoadLocation = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
LoadMetadataDetails[] listOfLoadFolderDetailsArray = null;
if (!FileFactory.isFileExist(dataLoadLocation, FileFactory.getFileType(dataLoadLocation))) {
// log error.
@@ -362,9 +354,7 @@ public class SegmentStatusManager {
if (carbonDeleteSegmentLock.lockWithRetries()) {
LOG.info("Delete segment lock has been successfully acquired");
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(
- identifier.getTablePath(), identifier.getCarbonTableIdentifier());
- String dataLoadLocation = carbonTablePath.getTableStatusFilePath();
+ String dataLoadLocation = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
LoadMetadataDetails[] listOfLoadFolderDetailsArray = null;
if (!FileFactory.isFileExist(dataLoadLocation, FileFactory.getFileType(dataLoadLocation))) {
@@ -684,7 +674,7 @@ public class SegmentStatusManager {
*/
public static Boolean checkIfAnyLoadInProgressForTable(CarbonTable carbonTable) {
Boolean loadInProgress = false;
- String metaPath = carbonTable.getMetaDataFilepath();
+ String metaPath = carbonTable.getMetadataPath();
LoadMetadataDetails[] listOfLoadFolderDetailsArray =
SegmentStatusManager.readLoadMetadata(metaPath);
if (listOfLoadFolderDetailsArray.length != 0) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 66f7a12..da70399 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -48,7 +48,6 @@ import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
import org.apache.carbondata.core.mutate.TupleIdEnum;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import com.google.gson.Gson;
@@ -64,24 +63,20 @@ public class SegmentUpdateStatusManager {
private static final LogService LOG =
LogServiceFactory.getLogService(SegmentUpdateStatusManager.class.getName());
- private AbsoluteTableIdentifier absoluteTableIdentifier;
+ private AbsoluteTableIdentifier identifier;
private LoadMetadataDetails[] segmentDetails;
private SegmentUpdateDetails[] updateDetails;
- private CarbonTablePath carbonTablePath;
private Map<String, SegmentUpdateDetails> blockAndDetailsMap;
/**
- * @param absoluteTableIdentifier
+ * @param identifier
*/
- public SegmentUpdateStatusManager(AbsoluteTableIdentifier absoluteTableIdentifier) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
- carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
- SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
+ public SegmentUpdateStatusManager(AbsoluteTableIdentifier identifier) {
+ this.identifier = identifier;
// current it is used only for read function scenarios, as file update always requires to work
// on latest file status.
- segmentDetails =
- segmentStatusManager.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
+ segmentDetails = SegmentStatusManager.readLoadMetadata(
+ CarbonTablePath.getMetadataPath(identifier.getTablePath()));
updateDetails = readLoadMetadata();
populateMap();
}
@@ -128,8 +123,6 @@ public class SegmentUpdateStatusManager {
}
-
-
/**
* Returns the LoadMetadata Details
* @return
@@ -160,7 +153,7 @@ public class SegmentUpdateStatusManager {
* @return
*/
public ICarbonLock getTableUpdateStatusLock() {
- return CarbonLockFactory.getCarbonLockObj(absoluteTableIdentifier,
+ return CarbonLockFactory.getCarbonLockObj(identifier,
LockUsage.TABLE_UPDATE_STATUS_LOCK);
}
@@ -176,7 +169,8 @@ public class SegmentUpdateStatusManager {
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
String endTimeStamp = "";
String startTimeStamp = "";
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
+ String segmentPath = CarbonTablePath.getSegmentPath(
+ identifier.getTablePath(), segmentId);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
for (LoadMetadataDetails eachSeg : segmentDetails) {
@@ -234,9 +228,7 @@ public class SegmentUpdateStatusManager {
* @throws Exception
*/
public String[] getDeleteDeltaFilePath(String blockFilePath) throws Exception {
- int tableFactPathLength = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier()).getFactDir().length() + 1;
+ int tableFactPathLength = CarbonTablePath.getFactDir(identifier.getTablePath()).length() + 1;
String blockame = blockFilePath.substring(tableFactPathLength);
String tupleId = CarbonTablePath.getShortBlockId(blockame);
return getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT)
@@ -253,11 +245,9 @@ public class SegmentUpdateStatusManager {
*/
public List<String> getDeltaFiles(String tupleId, String extension) throws Exception {
try {
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
String segment = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID);
- String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segment);
+ String carbonDataDirectoryPath = CarbonTablePath.getSegmentPath(
+ identifier.getTablePath(), segment);
String completeBlockName = CarbonTablePath.addDataPartPrefix(
CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCK_ID)
+ CarbonCommonConstants.FACT_FILE_EXT);
@@ -385,11 +375,8 @@ public class SegmentUpdateStatusManager {
*/
public CarbonFile[] getDeleteDeltaFilesList(final String segmentId, final String blockName) {
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
-
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
+ String segmentPath = CarbonTablePath.getSegmentPath(
+ identifier.getTablePath(), segmentId);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
@@ -436,16 +423,12 @@ public class SegmentUpdateStatusManager {
final String fileExtension, final boolean excludeOriginalFact,
CarbonFile[] allFilesOfSegment, boolean isAbortedFile) {
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
- SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
String endTimeStamp = "";
String startTimeStamp = "";
long factTimeStamp = 0;
- LoadMetadataDetails[] segmentDetails =
- segmentStatusManager.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
+ LoadMetadataDetails[] segmentDetails = SegmentStatusManager.readLoadMetadata(
+ CarbonTablePath.getMetadataPath(identifier.getTablePath()));
for (LoadMetadataDetails eachSeg : segmentDetails) {
if (eachSeg.getLoadName().equalsIgnoreCase(segmentId)) {
@@ -650,13 +633,9 @@ public class SegmentUpdateStatusManager {
return new SegmentUpdateDetails[0];
}
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
-
String tableUpdateStatusPath =
- carbonTablePath.getMetadataDirectoryPath() + CarbonCommonConstants.FILE_SEPARATOR
- + tableUpdateStatusIdentifier;
+ CarbonTablePath.getMetadataPath(identifier.getTablePath()) +
+ CarbonCommonConstants.FILE_SEPARATOR + tableUpdateStatusIdentifier;
AtomicFileOperations fileOperation = new AtomicFileOperationsImpl(tableUpdateStatusPath,
FileFactory.getFileType(tableUpdateStatusPath));
@@ -684,12 +663,9 @@ public class SegmentUpdateStatusManager {
* @return updateStatusFileName
*/
private String getUpdatedStatusIdentifier() {
- SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
LoadMetadataDetails[] loadDetails =
- ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
+ SegmentStatusManager.readLoadMetadata(
+ CarbonTablePath.getMetadataPath(identifier.getTablePath()));
if (loadDetails.length == 0) {
return null;
}
@@ -704,13 +680,9 @@ public class SegmentUpdateStatusManager {
*/
public void writeLoadDetailsIntoFile(List<SegmentUpdateDetails> listOfSegmentUpdateDetailsArray,
String updateStatusFileIdentifier) throws IOException {
-
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
-
String fileLocation =
- carbonTablePath.getMetadataDirectoryPath() + CarbonCommonConstants.FILE_SEPARATOR
+ CarbonTablePath.getMetadataPath(identifier.getTablePath())
+ + CarbonCommonConstants.FILE_SEPARATOR
+ CarbonUpdateUtil.getUpdateStatusFileName(updateStatusFileIdentifier);
AtomicFileOperations fileWrite =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index b16bc5e..80a382c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -85,7 +85,6 @@ import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.BlockletHeader;
import org.apache.carbondata.format.DataChunk2;
@@ -1065,20 +1064,18 @@ public final class CarbonUtil {
*
* @param taskId
* @param tableBlockInfoList
- * @param absoluteTableIdentifier
+ * @param identifier
*/
public static long calculateDriverBTreeSize(String taskId, String bucketNumber,
- List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier absoluteTableIdentifier) {
+ List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier identifier) {
// need to sort the block info list based for task in ascending order so
// it will be sinkup with block index read from file
Collections.sort(tableBlockInfoList);
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
// geting the index file path
//TODO need to pass proper partition number when partiton will be supported
- String carbonIndexFilePath = carbonTablePath
- .getCarbonIndexFilePath(taskId, tableBlockInfoList.get(0).getSegmentId(),
+ String carbonIndexFilePath = CarbonTablePath
+ .getCarbonIndexFilePath(identifier.getTablePath(), taskId,
+ tableBlockInfoList.get(0).getSegmentId(),
bucketNumber, CarbonTablePath.DataFileUtil
.getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
tableBlockInfoList.get(0).getVersion());
@@ -1305,23 +1302,21 @@ public final class CarbonUtil {
*
* @param taskId task id of the file
* @param tableBlockInfoList list of table block
- * @param absoluteTableIdentifier absolute table identifier
+ * @param identifier absolute table identifier
* @return list of block info
* @throws IOException if any problem while reading
*/
public static List<DataFileFooter> readCarbonIndexFile(String taskId, String bucketNumber,
- List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier absoluteTableIdentifier)
+ List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier identifier)
throws IOException {
// need to sort the block info list based for task in ascending order so
// it will be sinkup with block index read from file
Collections.sort(tableBlockInfoList);
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
// geting the index file path
//TODO need to pass proper partition number when partiton will be supported
- String carbonIndexFilePath = carbonTablePath
- .getCarbonIndexFilePath(taskId, tableBlockInfoList.get(0).getSegmentId(),
+ String carbonIndexFilePath = CarbonTablePath
+ .getCarbonIndexFilePath(identifier.getTablePath(), taskId,
+ tableBlockInfoList.get(0).getSegmentId(),
bucketNumber, CarbonTablePath.DataFileUtil
.getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
tableBlockInfoList.get(0).getVersion());
@@ -2177,21 +2172,6 @@ public final class CarbonUtil {
}
/**
- * get the parent folder of old table path and returns the new tablePath by appending new
- * tableName to the parent
- *
- * @param carbonTablePath Old tablePath
- * @param newTableName new table name
- * @return the new table path
- */
- public static String getNewTablePath(
- Path carbonTablePath,
- String newTableName) {
- Path parentPath = carbonTablePath.getParent();
- return parentPath.toString() + CarbonCommonConstants.FILE_SEPARATOR + newTableName;
- }
-
- /**
* This method will calculate the data size and index size for carbon table
*/
public static Map<String, Long> calculateDataIndexSize(CarbonTable carbonTable)
@@ -2201,18 +2181,17 @@ public final class CarbonUtil {
long indexSize = 0L;
long lastUpdateTime = 0L;
boolean needUpdate = false;
- AbsoluteTableIdentifier absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
+ AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
String isCalculated = CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.ENABLE_CALCULATE_SIZE,
CarbonCommonConstants.DEFAULT_ENABLE_CALCULATE_SIZE);
if (isCalculated.equalsIgnoreCase("true")) {
- SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
+ SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
try {
if (carbonLock.lockWithRetries()) {
LOGGER.info("Acquired lock for table for table status updation");
- String metadataPath = carbonTable.getMetaDataFilepath();
+ String metadataPath = carbonTable.getMetadataPath();
LoadMetadataDetails[] loadMetadataDetails =
SegmentStatusManager.readLoadMetadata(metadataPath);
@@ -2226,8 +2205,8 @@ public final class CarbonUtil {
if (null == dsize || null == isize) {
needUpdate = true;
LOGGER.info("It is an old segment, need calculate data size and index size again");
- HashMap<String, Long> map = CarbonUtil
- .getDataSizeAndIndexSize(carbonTablePath, loadMetadataDetail.getLoadName());
+ HashMap<String, Long> map = CarbonUtil.getDataSizeAndIndexSize(
+ identifier.getTablePath(), loadMetadataDetail.getLoadName());
dsize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE));
isize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE));
loadMetadataDetail.setDataSize(dsize);
@@ -2239,10 +2218,12 @@ public final class CarbonUtil {
}
// If it contains old segment, write new load details
if (needUpdate) {
- SegmentStatusManager.writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath(),
+ SegmentStatusManager.writeLoadDetailsIntoFile(
+ CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()),
loadMetadataDetails);
}
- String tableStatusPath = carbonTablePath.getTableStatusFilePath();
+ String tableStatusPath =
+ CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
if (FileFactory.isFileExist(tableStatusPath, FileFactory.getFileType(tableStatusPath))) {
lastUpdateTime =
FileFactory.getCarbonFile(tableStatusPath, FileFactory.getFileType(tableStatusPath))
@@ -2269,12 +2250,12 @@ public final class CarbonUtil {
}
// Get the total size of carbon data and the total size of carbon index
- public static HashMap<String, Long> getDataSizeAndIndexSize(CarbonTablePath carbonTablePath,
+ public static HashMap<String, Long> getDataSizeAndIndexSize(String tablePath,
String segmentId) throws IOException {
long carbonDataSize = 0L;
long carbonIndexSize = 0L;
HashMap<String, Long> dataAndIndexSize = new HashMap<String, Long>();
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
+ String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
FileFactory.FileType fileType = FileFactory.getFileType(segmentPath);
switch (fileType) {
case HDFS:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/util/path/CarbonSharedDictionaryPath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonSharedDictionaryPath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonSharedDictionaryPath.java
deleted file mode 100644
index b864e6e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonSharedDictionaryPath.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.util.path;
-
-import java.io.File;
-
-/**
- * Helps to get Shared dimension files path.
- */
-public class CarbonSharedDictionaryPath {
-
- private static final String SHAREDDIM_DIR = "SharedDictionary";
- private static final String DICTIONARY_EXT = ".dict";
- private static final String DICTIONARY_META_EXT = ".dictmeta";
- private static final String SORT_INDEX_EXT = ".sortindex";
-
- /***
- * @param storePath store path
- * @param databaseName data base name
- * @param columnId unique column identifier
- * @return absolute path of shared dictionary file
- */
- public static String getDictionaryFilePath(String storePath, String databaseName,
- String columnId) {
- return getSharedDictionaryDir(storePath, databaseName) + File.separator + columnId
- + DICTIONARY_EXT;
- }
-
- /***
- * @param storePath store path
- * @param databaseName data base name
- * @param columnId unique column identifier
- * @return absolute path of shared dictionary meta file
- */
- public static String getDictionaryMetaFilePath(String storePath, String databaseName,
- String columnId) {
- return getSharedDictionaryDir(storePath, databaseName) + File.separator + columnId
- + DICTIONARY_META_EXT;
- }
-
- /***
- * @param storePath store path
- * @param databaseName data base name
- * @param columnId unique column identifier
- * @return absolute path of shared dictionary sort index file
- */
- public static String getSortIndexFilePath(String storePath, String databaseName,
- String columnId) {
- return getSharedDictionaryDir(storePath, databaseName) + File.separator + columnId
- + SORT_INDEX_EXT;
- }
-
- private static String getSharedDictionaryDir(String storePath, String databaseName) {
- return storePath + File.separator + databaseName + File.separator + SHAREDDIM_DIR;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
deleted file mode 100644
index e57448d..0000000
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.util.path;
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-
-import org.apache.hadoop.fs.Path;
-
-/**
- * Helps to get Store content paths.
- */
-public class CarbonStorePath extends Path {
-
- private String storePath;
-
- public CarbonStorePath(String storePathString) {
- super(storePathString);
- this.storePath = storePathString;
- }
-
- /**
- * gets CarbonTablePath object to manage table paths
- *
- * @param tablePath the store path of the segment
- * @param tableIdentifier identifier of carbon table that the segment belong to
- * @return the store location of the segment
- */
- public static CarbonTablePath getCarbonTablePath(String tablePath,
- CarbonTableIdentifier tableIdentifier) {
- return new CarbonTablePath(tableIdentifier, tablePath);
- }
-
- public static CarbonTablePath getCarbonTablePath(AbsoluteTableIdentifier identifier) {
- CarbonTableIdentifier id = identifier.getCarbonTableIdentifier();
- return new CarbonTablePath(id, identifier.getTablePath());
- }
-
- /**
- * gets CarbonTablePath object to manage table paths
- */
- public CarbonTablePath getCarbonTablePath(CarbonTableIdentifier tableIdentifier) {
- return CarbonStorePath.getCarbonTablePath(storePath, tableIdentifier);
- }
-
- @Override public boolean equals(Object o) {
- if (!(o instanceof CarbonStorePath)) {
- return false;
- }
- CarbonStorePath path = (CarbonStorePath)o;
- return storePath.equals(path.storePath) && super.equals(o);
- }
-
- @Override public int hashCode() {
- return super.hashCode() + storePath.hashCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 36eaa56..d870f66 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -22,16 +22,14 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.hadoop.fs.Path;
-
/**
* Helps to get Table content paths.
*/
-public class CarbonTablePath extends Path {
+public class CarbonTablePath {
private static final String METADATA_DIR = "Metadata";
private static final String DICTIONARY_EXT = ".dict";
@@ -54,19 +52,10 @@ public class CarbonTablePath extends Path {
private static final String STREAMING_LOG_DIR = "log";
private static final String STREAMING_CHECKPOINT_DIR = "checkpoint";
- private String tablePath;
- private CarbonTableIdentifier carbonTableIdentifier;
-
/**
- * structure CarbonTablePath object to manage table paths
- *
- * @param carbonTableIdentifier identifier of carbon table that the segment belong to
- * @param tablePathString the store path of the segment
+ * This class provides static utility only.
*/
- public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String tablePathString) {
- super(tablePathString);
- this.carbonTableIdentifier = carbonTableIdentifier;
- this.tablePath = tablePathString;
+ private CarbonTablePath() {
}
/**
@@ -136,120 +125,74 @@ public class CarbonTablePath extends Path {
}
/**
- * gets table path
- */
- public String getPath() {
- return tablePath;
- }
-
- /**
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
+ * Return absolute path of dictionary file
*/
- public String getDictionaryFilePath(String columnId) {
- return getMetaDataDir() + File.separator + getDictionaryFileName(columnId);
+ public static String getDictionaryFilePath(String tablePath, String columnId) {
+ return getMetadataPath(tablePath) + File.separator + getDictionaryFileName(columnId);
}
/**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
+ * Return absolute path of dictionary file
*/
- public String getDictionaryFilePath(String dictionaryPath, String columnId) {
+ public static String getExternalDictionaryFilePath(String dictionaryPath, String columnId) {
return dictionaryPath + File.separator + getDictionaryFileName(columnId);
}
/**
- * This method will return the metadata directory location for a table
- *
- * @return
- */
- public String getMetadataDirectoryPath() {
- return getMetaDataDir();
- }
-
- /**
- * Return metadata path based on `tablePath`
+ * Return metadata path
*/
public static String getMetadataPath(String tablePath) {
return tablePath + File.separator + METADATA_DIR;
}
/**
- * @param columnId unique column identifier
- * @return absolute path of dictionary meta file
- */
- public String getDictionaryMetaFilePath(String columnId) {
- return getMetaDataDir() + File.separator + columnId + DICTIONARY_META_EXT;
- }
-
- /**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
+ * Return absolute path of dictionary meta file
*/
- public String getDictionaryMetaFilePath(String dictionaryPath, String columnId) {
+ public static String getExternalDictionaryMetaFilePath(String dictionaryPath, String columnId) {
return dictionaryPath + File.separator + columnId + DICTIONARY_META_EXT;
}
/**
- * @param columnId unique column identifier
- * @return absolute path of sort index file
+ * Return absolute path of dictionary meta file
*/
- public String getSortIndexFilePath(String columnId) {
- return getMetaDataDir() + File.separator + columnId + SORT_INDEX_EXT;
+ public static String getDictionaryMetaFilePath(String tablePath, String columnId) {
+ return getMetadataPath(tablePath) + File.separator + columnId + DICTIONARY_META_EXT;
}
/**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
+ * Return absolute path of sort index file
*/
- public String getSortIndexFilePath(String dictionaryPath, String columnId) {
- return dictionaryPath + File.separator + columnId + SORT_INDEX_EXT;
+ public static String getSortIndexFilePath(String tablePath, String columnId) {
+ return getMetadataPath(tablePath) + File.separator + columnId + SORT_INDEX_EXT;
}
/**
- *
- * @param columnId
- * @param dictOffset
- * @return absolute path of sortindex with appeneded dictionary offset
+ * Return sortindex file path based on specified dictionary path
*/
- public String getSortIndexFilePath(String columnId, long dictOffset) {
- return getMetaDataDir() + File.separator + columnId + "_" + dictOffset + SORT_INDEX_EXT;
+ public static String getExternalSortIndexFilePath(String dictionaryPath, String columnId) {
+ return dictionaryPath + File.separator + columnId + SORT_INDEX_EXT;
}
/**
- * @param dictionaryPath
- * @param columnId unique column identifier
- * @param dictOffset
- * @return absolute path of dictionary file
+ * Return sortindex file path for columnId and offset based on specified dictionary path
*/
- public String getSortIndexFilePath(String dictionaryPath, String columnId, long dictOffset) {
+ public static String getExternalSortIndexFilePath(String dictionaryPath, String columnId,
+ long dictOffset) {
return dictionaryPath + File.separator + columnId + "_" + dictOffset + SORT_INDEX_EXT;
}
/**
- * @return absolute path of schema file
- */
- public String getSchemaFilePath() {
- return getMetaDataDir() + File.separator + SCHEMA_FILE;
- }
-
- /**
- * return the schema file path
- * @param tablePath path to table files
- * @return schema file path
+ * Return absolute path of schema file
*/
public static String getSchemaFilePath(String tablePath) {
- return tablePath + File.separator + METADATA_DIR + File.separator + SCHEMA_FILE;
+ return getMetadataPath(tablePath) + File.separator + SCHEMA_FILE;
}
/**
- * @return absolute path of table status file
+ * Return absolute path of table status file
*/
- public String getTableStatusFilePath() {
- return getMetaDataDir() + File.separator + TABLE_STATUS_FILE;
+ public static String getTableStatusFilePath(String tablePath) {
+ return getMetadataPath(tablePath) + File.separator + TABLE_STATUS_FILE;
}
/**
@@ -260,9 +203,9 @@ public class CarbonTablePath extends Path {
* @param factUpdateTimeStamp unique identifier to identify an update
* @return absolute path of data file stored in carbon data format
*/
- public String getCarbonDataFilePath(String segmentId, Integer filePartNo, Long taskNo,
- int batchNo, int bucketNumber, String factUpdateTimeStamp) {
- return getSegmentDir(segmentId) + File.separator + getCarbonDataFileName(
+ public static String getCarbonDataFilePath(String tablePath, String segmentId, Integer filePartNo,
+ Long taskNo, int batchNo, int bucketNumber, String factUpdateTimeStamp) {
+ return getSegmentPath(tablePath, segmentId) + File.separator + getCarbonDataFileName(
filePartNo, taskNo, bucketNumber, batchNo, factUpdateTimeStamp);
}
@@ -274,9 +217,9 @@ public class CarbonTablePath extends Path {
* @param segmentId segment number
* @return full qualified carbon index path
*/
- public String getCarbonIndexFilePath(final String taskId, final String segmentId,
- final String bucketNumber) {
- String segmentDir = getSegmentDir(segmentId);
+ private static String getCarbonIndexFilePath(final String tablePath, final String taskId,
+ final String segmentId, final String bucketNumber) {
+ String segmentDir = getSegmentPath(tablePath, segmentId);
CarbonFile carbonFile =
FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
@@ -309,27 +252,28 @@ public class CarbonTablePath extends Path {
* timestamp
* @return carbon index file path
*/
- public String getCarbonIndexFilePath(String taskId, String segmentId, String bucketNumber,
- String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
+ public static String getCarbonIndexFilePath(String tablePath, String taskId, String segmentId,
+ String bucketNumber, String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
switch (columnarFormatVersion) {
case V1:
case V2:
- return getCarbonIndexFilePath(taskId, segmentId, bucketNumber);
+ return getCarbonIndexFilePath(tablePath, taskId, segmentId, bucketNumber);
default:
- String segmentDir = getSegmentDir(segmentId);
+ String segmentDir = getSegmentPath(tablePath, segmentId);
return segmentDir + File.separator + getCarbonIndexFileName(taskId,
Integer.parseInt(bucketNumber), timeStamp);
}
}
- public String getCarbonIndexFilePath(String taskId, String segmentId, int batchNo,
- String bucketNumber, String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
+ public static String getCarbonIndexFilePath(String tablePath, String taskId, String segmentId,
+ int batchNo, String bucketNumber, String timeStamp,
+ ColumnarFormatVersion columnarFormatVersion) {
switch (columnarFormatVersion) {
case V1:
case V2:
- return getCarbonIndexFilePath(taskId, segmentId, bucketNumber);
+ return getCarbonIndexFilePath(tablePath, taskId, segmentId, bucketNumber);
default:
- String segmentDir = getSegmentDir(segmentId);
+ String segmentDir = getSegmentPath(tablePath, segmentId);
return segmentDir + File.separator + getCarbonIndexFileName(Long.parseLong(taskId),
Integer.parseInt(bucketNumber), batchNo, timeStamp);
}
@@ -341,13 +285,10 @@ public class CarbonTablePath extends Path {
}
/**
- * Gets absolute path of data file
- *
- * @param segmentId unique partition identifier
- * @return absolute path of data file stored in carbon data format
+ * Return the segment path from table path and segmentid
*/
- public String getCarbonDataDirectoryPath(String segmentId) {
- return getSegmentDir(segmentId);
+ public static String getSegmentPath(String tablePath, String segmentId) {
+ return getPartitionDir(tablePath) + File.separator + SEGMENT_PREFIX + segmentId;
}
/**
@@ -385,48 +326,46 @@ public class CarbonTablePath extends Path {
return segmentDir + File.separator + getCarbonStreamIndexFileName();
}
- public String getSegmentDir(String segmentId) {
- return getPartitionDir() + File.separator + SEGMENT_PREFIX + segmentId;
- }
-
// This partition is not used in any code logic, just keep backward compatibility
public static final String DEPRECATED_PATITION_ID = "0";
- public String getPartitionDir() {
- return getFactDir() + File.separator + PARTITION_PREFIX +
- CarbonTablePath.DEPRECATED_PATITION_ID;
+ /**
+ * Return true if tablePath exists
+ */
+ public static boolean exists(String tablePath) {
+ return FileFactory.getCarbonFile(tablePath, FileFactory.getFileType(tablePath)).exists();
}
- private String getMetaDataDir() {
- return tablePath + File.separator + METADATA_DIR;
+ public static String getPartitionDir(String tablePath) {
+ return getFactDir(tablePath) + File.separator + PARTITION_PREFIX +
+ CarbonTablePath.DEPRECATED_PATITION_ID;
}
- public String getFactDir() {
+ public static String getFactDir(String tablePath) {
return tablePath + File.separator + FACT_DIR;
}
- public String getStreamingLogDir() {
+ public static String getStreamingLogDir(String tablePath) {
return tablePath + File.separator + STREAMING_DIR + File.separator + STREAMING_LOG_DIR;
}
- public String getStreamingCheckpointDir() {
+ public static String getStreamingCheckpointDir(String tablePath) {
return tablePath + File.separator + STREAMING_DIR + File.separator + STREAMING_CHECKPOINT_DIR;
}
- public CarbonTableIdentifier getCarbonTableIdentifier() {
- return carbonTableIdentifier;
- }
-
- @Override public boolean equals(Object o) {
- if (!(o instanceof CarbonTablePath)) {
- return false;
- }
- CarbonTablePath path = (CarbonTablePath) o;
- return tablePath.equals(path.tablePath) && super.equals(o);
- }
-
- @Override public int hashCode() {
- return super.hashCode() + tablePath.hashCode();
+ /**
+ * get the parent folder of old table path and returns the new tablePath by appending new
+ * tableName to the parent
+ *
+ * @param tablePath Old tablePath
+ * @param newTableName new table name
+ * @return the new table path
+ */
+ public static String getNewTablePath(
+ String tablePath,
+ String newTableName) {
+ Path parentPath = new Path(tablePath).getParent();
+ return parentPath.toString() + CarbonCommonConstants.FILE_SEPARATOR + newTableName;
}
/**
@@ -445,11 +384,8 @@ public class CarbonTablePath extends Path {
return fileName.substring(startIndex, endIndex);
}
-
/**
- * This will return the timestamp present in the delete delta file.
- * @param fileName
- * @return
+ * Return the timestamp present in the delete delta file.
*/
public static String getTimeStampFromDeleteDeltaFile(String fileName) {
return fileName.substring(fileName.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
@@ -457,9 +393,7 @@ public class CarbonTablePath extends Path {
}
/**
- * This will return the timestamp present in the delete delta file.
- * @param fileName
- * @return
+ * Return the timestamp present in the delete delta file.
*/
public static String getBlockNameFromDeleteDeltaFile(String fileName) {
return fileName.substring(0,
@@ -467,7 +401,7 @@ public class CarbonTablePath extends Path {
}
/**
- * gets updated timestamp information from given carbon data file name
+ * Return the updated timestamp information from given carbon data file name
*/
public static String getBucketNo(String carbonFilePath) {
// Get the file name from path
@@ -485,7 +419,7 @@ public class CarbonTablePath extends Path {
}
/**
- * gets file part number information from given carbon data file name
+ * Return the file part number information from given carbon data file name
*/
public static String getPartNo(String carbonDataFileName) {
// Get the file name from path
@@ -497,7 +431,7 @@ public class CarbonTablePath extends Path {
}
/**
- * gets updated timestamp information from given carbon data file name
+ * Return the updated timestamp information from given carbon data file name
*/
public static String getTaskNo(String carbonDataFileName) {
// Get the file name from path
@@ -510,35 +444,30 @@ public class CarbonTablePath extends Path {
}
/**
- * get the taskId part from taskNo(include taskId + batchNo)
- * @param taskNo
- * @return
+ * Return the taskId part from taskNo(include taskId + batchNo)
*/
public static long getTaskIdFromTaskNo(String taskNo) {
return Long.parseLong(taskNo.split(BATCH_PREFIX)[0]);
}
+ /**
+ * Return the batch number from taskNo string
+ */
public static int getBatchNoFromTaskNo(String taskNo) {
return Integer.parseInt(taskNo.split(BATCH_PREFIX)[1]);
}
/**
- * Gets the file name from file path
+ * Return the file name from file path
*/
- private static String getFileName(String carbonDataFileName) {
- int endIndex = carbonDataFileName.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
+ private static String getFileName(String dataFilePath) {
+ int endIndex = dataFilePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
if (endIndex > -1) {
- return carbonDataFileName.substring(endIndex + 1, carbonDataFileName.length());
+ return dataFilePath.substring(endIndex + 1, dataFilePath.length());
} else {
- return carbonDataFileName;
+ return dataFilePath;
}
}
- }
-
- /**
- * To manage data path and composition
- */
- public static class DataPathUtil {
/**
* gets segement id from given absolute data file path
@@ -546,11 +475,11 @@ public class CarbonTablePath extends Path {
public static String getSegmentId(String dataFileAbsolutePath) {
// find segment id from last of data file path
String tempdataFileAbsolutePath = dataFileAbsolutePath.replace(
- CarbonCommonConstants.WINDOWS_FILE_SEPARATOR, CarbonCommonConstants.FILE_SEPARATOR);
+ CarbonCommonConstants.WINDOWS_FILE_SEPARATOR, CarbonCommonConstants.FILE_SEPARATOR);
int endIndex = tempdataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
// + 1 for size of "/"
int startIndex = tempdataFileAbsolutePath.lastIndexOf(
- CarbonCommonConstants.FILE_SEPARATOR, endIndex - 1) + 1;
+ CarbonCommonConstants.FILE_SEPARATOR, endIndex - 1) + 1;
String segmentDirStr = dataFileAbsolutePath.substring(startIndex, endIndex);
//identify id in segment_<id>
String[] segmentDirSplits = segmentDirStr.split("_");
@@ -582,19 +511,16 @@ public class CarbonTablePath extends Path {
}
/**
- * returns the carbondata file name
- *
- * @param carbonDataFilePath carbondata file path
- * @return
+ * Return the carbondata file name
*/
public static String getCarbonDataFileName(String carbonDataFilePath) {
- return carbonDataFilePath
- .substring(carbonDataFilePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1,
- carbonDataFilePath.indexOf(CARBON_DATA_EXT));
+ return carbonDataFilePath.substring(
+ carbonDataFilePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1,
+ carbonDataFilePath.indexOf(CARBON_DATA_EXT));
}
/**
- * @return prefix of carbon data
+ * Return prefix of carbon data
*/
public static String getCarbonDataPrefix() {
return DATA_PART_PREFIX;
@@ -638,40 +564,6 @@ public class CarbonTablePath extends Path {
}
/**
- * This method will append strings in path and return block id
- *
- * @param shortBlockId
- * @return blockId
- */
- public static String getBlockId(String shortBlockId) {
- String[] splitRecords = shortBlockId.split(CarbonCommonConstants.FILE_SEPARATOR);
- StringBuffer sb = new StringBuffer();
- for (int i = 0; i < splitRecords.length; i++) {
- if (i == 0) {
- sb.append(PARTITION_PREFIX);
- sb.append(splitRecords[i]);
- } else if (i == 1) {
- sb.append(CarbonCommonConstants.FILE_SEPARATOR);
- sb.append(SEGMENT_PREFIX);
- sb.append(splitRecords[i]);
- } else if (i == 2) {
- sb.append(CarbonCommonConstants.FILE_SEPARATOR);
- sb.append(DATA_PART_PREFIX);
- sb.append(splitRecords[i]);
- } else if (i == 3) {
- sb.append(CarbonCommonConstants.FILE_SEPARATOR);
- sb.append(splitRecords[i]);
- sb.append(CARBON_DATA_EXT);
- } else {
- sb.append(CarbonCommonConstants.FILE_SEPARATOR);
- sb.append(splitRecords[i]);
- }
- }
- return sb.toString();
- }
-
-
- /**
* adds data part prefix to given value
* @return partition prefix
*/
@@ -700,10 +592,4 @@ public class CarbonTablePath extends Path {
+ "-" + DataFileUtil.getTimeStampFromFileName(actualBlockName) + INDEX_FILE_EXT;
}
- /**
- * Get the segment path from table path and segmentid
- */
- public static String getSegmentPath(String tablePath, String segmentId) {
- return tablePath + "/Fact/Part0/Segment_" + segmentId;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
index 31e44a2..7d829b9 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -48,7 +47,7 @@ public class AbstractDictionaryCacheTest {
protected CarbonTableIdentifier carbonTableIdentifier;
- protected AbsoluteTableIdentifier absoluteTableIdentifier;
+ protected AbsoluteTableIdentifier identifier;
protected String databaseName;
@@ -107,7 +106,7 @@ public class AbstractDictionaryCacheTest {
protected DictionaryColumnUniqueIdentifier createDictionaryColumnUniqueIdentifier(
String columnId) {
ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, DataTypes.STRING);
- return new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier,
+ return new DictionaryColumnUniqueIdentifier(identifier, columnIdentifier,
DataTypes.STRING);
}
@@ -130,13 +129,11 @@ public class AbstractDictionaryCacheTest {
throws IOException {
ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, null);
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
- new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier,
+ new DictionaryColumnUniqueIdentifier(identifier, columnIdentifier,
columnIdentifier.getDataType());
CarbonDictionaryWriter carbonDictionaryWriter =
new CarbonDictionaryWriterImpl(dictionaryColumnUniqueIdentifier);
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- CarbonUtil.checkAndCreateFolder(carbonTablePath.getMetadataDirectoryPath());
+ CarbonUtil.checkAndCreateFolder(CarbonTablePath.getMetadataPath(identifier.getTablePath()));
List<byte[]> valueList = convertStringListToByteArray(data);
try {
carbonDictionaryWriter.write(valueList);
[03/50] [abbrv] carbondata git commit:
[CARBONDATA-2088][CARBONDATA-1516] Optimize syntax for creating timeseries
pre-aggregate table
Posted by ra...@apache.org.
[CARBONDATA-2088][CARBONDATA-1516] Optimize syntax for creating timeseries pre-aggregate table
change using 'timeseries' instead of using preaggregate for creating timeseries pre-aggregate table
change timeseries.eventTime and hour_granularity and so on
granularity only support one
It should throw UnsupportDataMapException if don't use timeseries or preaggregate to create datamap
This closes #1865
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/181c280b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/181c280b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/181c280b
Branch: refs/heads/fgdatamap
Commit: 181c280b7d33ac5e4029bd935d6260b0fe79a2bf
Parents: b421c24
Author: xubo245 <60...@qq.com>
Authored: Fri Jan 26 17:47:46 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jan 30 17:25:19 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 -
.../schema/datamap/DataMapProvider.java | 32 +++
.../metadata/schema/datamap/Granularity.java | 46 +++
.../preaggregate/TestPreAggCreateCommand.scala | 60 +++-
.../TestPreAggregateTableSelection.scala | 26 +-
.../timeseries/TestTimeSeriesCreateTable.scala | 280 ++++++++++++++++---
.../timeseries/TestTimeseriesCompaction.scala | 67 ++++-
.../timeseries/TestTimeseriesDataLoad.scala | 142 +++++++++-
.../TestTimeseriesTableSelection.scala | 114 +++++++-
.../testsuite/datamap/TestDataMapCommand.scala | 155 +++++-----
.../MalformedDataMapCommandException.java | 32 +++
.../datamap/CarbonCreateDataMapCommand.scala | 72 ++---
.../CreatePreAggregateTableCommand.scala | 4 +-
.../command/timeseries/TimeSeriesUtil.scala | 83 +++++-
.../carbondata/CarbonDataSourceSuite.scala | 8 +
15 files changed, 943 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index f46feef..cf95dd9 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1543,10 +1543,6 @@ public final class CarbonCommonConstants {
*/
public static final long HANDOFF_SIZE_DEFAULT = 1024L * 1024 * 1024;
- public static final String TIMESERIES_EVENTTIME = "timeseries.eventtime";
-
- public static final String TIMESERIES_HIERARCHY = "timeseries.hierarchy";
-
/**
* It allows queries on hive metastore directly along with filter information, otherwise first
* fetches all partitions from hive and apply filters on it.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
new file mode 100644
index 0000000..65578b1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.datamap;
+
+/**
+ * type for create datamap
+ * The syntax of datamap creation is as follows.
+ * CREATE DATAMAP IF NOT EXISTS dataMapName ON TABLE tableName USING 'DataMapProvider'
+ * DMPROPERTIES('KEY'='VALUE') AS SELECT COUNT(COL1) FROM tableName
+ *
+ * Please refer {{org.apache.spark.sql.parser.CarbonSpark2SqlParser}}
+ */
+
+public enum DataMapProvider {
+ PREAGGREGATE,
+ TIMESERIES;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/Granularity.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/Granularity.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/Granularity.java
new file mode 100644
index 0000000..d6aefb6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/Granularity.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.datamap;
+
+/**
+ * type for create datamap
+ * The syntax of datamap creation is as follows.
+ * CREATE DATAMAP IF NOT EXISTS dataMapName ON TABLE tableName USING 'DataMapProvider'
+ * DMPROPERTIES('KEY'='VALUE') AS SELECT COUNT(COL1) FROM tableName
+ *
+ * Please refer {{org.apache.spark.sql.parser.CarbonSpark2SqlParser}}
+ */
+
+public enum Granularity {
+ YEAR("year_granularity"),
+ MONTH("month_granularity"),
+ DAY("day_granularity"),
+ HOUR("hour_granularity"),
+ MINUTE("minute_granularity"),
+ SECOND("second_granularity");
+ private String name;
+
+ Granularity(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index 755a449..d3f0ff8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -1,15 +1,18 @@
package org.apache.carbondata.integration.spark.testsuite.preaggregate
+import scala.collection.JavaConverters._
+
import org.apache.spark.sql.CarbonDatasourceHadoopRelation
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import scala.collection.JavaConverters._
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
@@ -212,6 +215,60 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
sql("drop datamap agg0 on table maintable")
}
+ val timeSeries = TIMESERIES.toString
+
+ test("test PreAggregate table selection: create with preaggregate and hierarchy") {
+ sql("DROP TABLE IF EXISTS maintabletime")
+ sql(
+ """
+ | CREATE TABLE maintabletime(year INT,month INT,name STRING,salary INT,dob STRING)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES(
+ | 'SORT_SCOPE'='Global_sort',
+ | 'TABLE_BLOCKSIZE'='23',
+ | 'SORT_COLUMNS'='month,year,name')
+ """.stripMargin)
+ sql("INSERT INTO maintabletime SELECT 10,11,'x',12,'2014-01-01 00:00:00'")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0 ON TABLE maintabletime
+ | USING 'preaggregate'
+ | AS SELECT dob,name FROM maintabletime
+ | GROUP BY dob,name
+ """.stripMargin)
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg1 ON TABLE maintabletime
+ | USING 'preaggregate'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dob',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dob,name FROM maintabletime
+ | GROUP BY dob,name
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains(s"$timeSeries keyword missing"))
+ sql("DROP TABLE IF EXISTS maintabletime")
+ }
+
+ test("test pre agg create table 21: using") {
+ sql("DROP DATAMAP agg0 ON TABLE maintable")
+
+ val e: Exception = intercept[Exception] {
+ sql(
+ """
+ | CREATE DATAMAP agg0 ON TABLE mainTable
+ | USING 'abc'
+ | AS SELECT column3, SUM(column3),column5, SUM(column5)
+ | FROM maintable
+ | GROUP BY column3,column5,column2
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains(
+ s"Unknown data map type abc"))
+ sql("DROP DATAMAP agg0 ON TABLE maintable")
+ }
def getCarbontable(plan: LogicalPlan) : CarbonTable ={
var carbonTable : CarbonTable = null
@@ -239,5 +296,6 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists PreAggMain")
sql("drop table if exists PreAggMain1")
sql("drop table if exists PreAggMain2")
+ sql("drop table if exists maintabletime")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 17d95ef..f9ac354 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -23,6 +23,8 @@ import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+
class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
@@ -267,6 +269,8 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
preAggTableValidator(df.queryExecution.analyzed, "maintable")
}
+ val timeSeries = TIMESERIES.toString
+
test("test PreAggregate table selection with timeseries and normal together") {
sql("drop table if exists maintabletime")
sql(
@@ -277,17 +281,26 @@ test("test PreAggregate table selection with timeseries and normal together") {
sql(
"create datamap agg0 on table maintabletime using 'preaggregate' as select dob,name from " +
"maintabletime group by dob,name")
- sql(
- "create datamap agg1 on table maintabletime using 'preaggregate' DMPROPERTIES ('timeseries" +
- ".eventTime'='dob', 'timeseries.hierarchy'='hour=1,day=1,month=1,year=1') as select dob," +
- "name from maintabletime group by dob,name")
- val df = sql("select timeseries(dob,'year') from maintabletime group by timeseries(dob,'year')")
+
+ sql(
+ s"""
+ | CREATE DATAMAP agg1_year ON TABLE maintabletime
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dob',
+ | 'YEAR_GRANULARITY'='1')
+ | AS SELECT dob, name FROM maintabletime
+ | GROUP BY dob,name
+ """.stripMargin)
+
+ val df = sql("SELECT timeseries(dob,'year') FROM maintabletime GROUP BY timeseries(dob,'year')")
preAggTableValidator(df.queryExecution.analyzed, "maintabletime_agg1_year")
+ sql("DROP TABLE IF EXISTS maintabletime")
}
test("test table selection when unsupported aggregate function is present") {
- sql("drop table if exists maintabletime")
+ sql("DROP TABLE IF EXISTS maintabletime")
sql(
"create table maintabletime(year int,month int,name string,salary int,dob string) stored" +
" by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23'," +
@@ -302,6 +315,7 @@ test("test PreAggregate table selection with timeseries and normal together") {
override def afterAll: Unit = {
sql("drop table if exists mainTable")
sql("drop table if exists lineitem")
+ sql("DROP TABLE IF EXISTS maintabletime")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
index c9041fa..0ca7cb9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
@@ -17,14 +17,68 @@
package org.apache.carbondata.integration.spark.testsuite.timeseries
import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.{BeforeAndAfterAll, Ignore}
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.spark.exception.{MalformedDataMapCommandException, MalformedCarbonCommandException}
class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
+ val timeSeries = TIMESERIES.toString
+
override def beforeAll: Unit = {
- sql("drop table if exists mainTable")
+ sql("DROP TABLE IF EXISTS mainTable")
sql("CREATE TABLE mainTable(dataTime timestamp, name string, city string, age int) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='second=1,hour=1,day=1,month=1,year=1') as select dataTime, sum(age) from mainTable group by dataTime")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_hour ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_day ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'day_granularity'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'month_granularity'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_year ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'year_granularity'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
test("test timeseries create table Zero") {
@@ -53,28 +107,29 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
intercept[Exception] {
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
| DMPROPERTIES (
- | 'timeseries.eventTime'='dataTime',
- | 'timeseries.hierarchy'='sec=1,hour=1,day=1,month=1,year=1')
- | as select dataTime, sum(age) from mainTable
- | group by dataTime
- """.stripMargin)
+ | 'EVENT_TIME'='dataTime',
+ | 'SEC_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
}
test("test timeseries create table Six") {
intercept[Exception] {
sql(
- """
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
- | DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='hour=2')
- | as select dataTime, sum(age) from mainTable
- | group by dataTime
- """.stripMargin)
-
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='2')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
}
@@ -82,24 +137,24 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
intercept[Exception] {
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
| DMPROPERTIES (
- | 'timeseries.eventTime'='dataTime',
- | 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
- | as select dataTime, sum(age) from mainTable
- | group by dataTime
- """.stripMargin)
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
| DMPROPERTIES (
- | 'timeseries.eventTime'='dataTime',
- | 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
- | as select dataTime, sum(age) from mainTable
- | group by dataTime
- """.stripMargin)
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
}
@@ -107,12 +162,14 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
intercept[Exception] {
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
- | DMPROPERTIES ('timeseries.eventTime'='name', 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
- | as select name, sum(age) from mainTable
- | group by name
- """.stripMargin)
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='name',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
}
@@ -120,18 +177,149 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
intercept[Exception] {
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='name',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ }
+
+ test("test timeseries create table: USING") {
+ val e: Exception = intercept[MalformedDataMapCommandException] {
+ sql(
+ """CREATE DATAMAP agg1 ON TABLE mainTable
+ | USING 'abc'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Unknown data map type abc"))
+ }
+
+ test("test timeseries create table: USING and catch MalformedCarbonCommandException") {
+ val e: Exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """CREATE DATAMAP agg1 ON TABLE mainTable
+ | USING 'abc'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Unknown data map type abc"))
+ }
+
+ test("test timeseries create table: Only one granularity level can be defined 1") {
+ val e: Exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1',
+ | 'HOUR_GRANULARITY'='1',
+ | 'DAY_GRANULARITY'='1',
+ | 'MONTH_GRANULARITY'='1',
+ | 'YEAR_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Only one granularity level can be defined"))
+ }
+
+ test("test timeseries create table: Only one granularity level can be defined 2") {
+ val e: Exception = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Only one granularity level can be defined"))
+ }
+
+ test("test timeseries create table: Only one granularity level can be defined 3") {
+ val e: Exception = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'DAY_GRANULARITY'='1',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Only one granularity level can be defined"))
+ }
+
+ test("test timeseries create table: Granularity only support 1") {
+ val e = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'DAY_GRANULARITY'='2')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Granularity only support 1"))
+ }
+
+ test("test timeseries create table: Granularity only support 1 and throw Exception") {
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='2')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Granularity only support 1"))
+ }
+
+ test("test timeseries create table: timeSeries should define time granularity") {
+ val e = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
| DMPROPERTIES (
- | 'timeseries.eventTime'='dataTime',
- | 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
- | as select name, sum(age) from mainTable
- | group by name
- """.stripMargin)
+ | 'EVENT_TIME'='dataTime')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
+ assert(e.getMessage.equals(s"$timeSeries should define time granularity"))
}
override def afterAll: Unit = {
- sql("drop table if exists mainTable")
+ sql("DROP TABLE IF EXISTS mainTable")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
index a410fe4..d66c402 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
@@ -18,15 +18,18 @@ package org.apache.carbondata.integration.spark.testsuite.timeseries
import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.util.SparkUtil4Test
-import org.scalatest.{BeforeAndAfterAll, Ignore}
+import org.scalatest.BeforeAndAfterAll
import org.scalatest.Matchers._
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
import org.apache.carbondata.core.util.CarbonProperties
class TestTimeseriesCompaction extends QueryTest with BeforeAndAfterAll {
var isCompactionEnabled = false
+ val timeSeries = TIMESERIES.toString
+
override def beforeAll: Unit = {
SparkUtil4Test.createTaskMockUp(sqlContext)
CarbonProperties.getInstance()
@@ -38,7 +41,67 @@ class TestTimeseriesCompaction extends QueryTest with BeforeAndAfterAll {
.addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
sql("drop table if exists mainTable")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_minute ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'MINUTE_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_hour ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_day ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'DAY_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_year ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'YEAR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
index d25710c..8bcdfc9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
@@ -24,10 +24,13 @@ import org.apache.spark.util.SparkUtil4Test
import org.scalatest.{BeforeAndAfterAll, Ignore}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
import org.apache.carbondata.core.util.CarbonProperties
class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
+ val timeSeries = TIMESERIES.toString
+
override def beforeAll: Unit = {
SparkUtil4Test.createTaskMockUp(sqlContext)
CarbonProperties.getInstance()
@@ -35,11 +38,131 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists mainTable")
sql("drop table if exists table_03")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_minute ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'minute_granularity'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_hour ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_day ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'DAY_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_year ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'year_granularity'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
sql("CREATE TABLE table_03 (imei string,age int,mac string,productdate timestamp,updatedate timestamp,gamePointId double,contractid double ) STORED BY 'org.apache.carbondata.format'")
sql(s"LOAD DATA inpath '$resourcesPath/data_sort.csv' INTO table table_03 options ('DELIMITER'=',', 'QUOTECHAR'='','FILEHEADER'='imei,age,mac,productdate,updatedate,gamePointId,contractid')")
- sql("create datamap ag1 on table table_03 using 'preaggregate' DMPROPERTIES ( 'timeseries.eventtime'='productdate','timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1')as select productdate,mac,sum(age) from table_03 group by productdate,mac")
+
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_second ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_minute ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'minute_granularity'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_hour ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_day ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'DAY_GRANULARITY'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_month ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'month_granularity'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_year ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'year_granularity'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
}
test("test Year level timeseries data validation1 ") {
@@ -93,12 +216,21 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
Row(Timestamp.valueOf("2016-02-23 01:02:50.0"),50)))
}
- test("test if timeseries load is successful on table creation") {
+ test("test if timeseries load is successful ON TABLE creation") {
sql("drop table if exists mainTable")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
- checkAnswer( sql("select * from maintable_agg0_second"),
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ checkAnswer( sql("select * FROM maintable_agg0_second"),
Seq(Row(Timestamp.valueOf("2016-02-23 01:01:30.0"),10),
Row(Timestamp.valueOf("2016-02-23 01:01:40.0"),20),
Row(Timestamp.valueOf("2016-02-23 01:01:50.0"),30),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
index a9d3965..3065952 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
@@ -24,14 +24,78 @@ import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.util.SparkUtil4Test
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
+ val timeSeries = TIMESERIES.toString
+
override def beforeAll: Unit = {
SparkUtil4Test.createTaskMockUp(sqlContext)
sql("drop table if exists mainTable")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_minute ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'minute_granularity'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_hour ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_day ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'DAY_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_year ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'YEAR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
}
@@ -100,6 +164,54 @@ class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
preAggTableValidator(df.queryExecution.analyzed,"maintable")
}
+ test("test timeseries table selection 14: Granularity only support 1 and throw Exception") {
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg3_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='2')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains("Granularity only support 1"))
+ }
+
+ test("test timeseries table selection 15: Granularity only support 1 and throw Exception") {
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg3_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='1.5')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains("Granularity only support 1"))
+ }
+
+ test("test timeseries table selection 16: Granularity only support 1 and throw Exception") {
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg3_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='-1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains("Granularity only support 1"))
+ }
+
def preAggTableValidator(plan: LogicalPlan, actualTableName: String) : Unit ={
var isValidPlan = false
plan.transform {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index a0ea317..0c38239 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.spark.exception.MalformedDataMapCommandException
class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
@@ -39,37 +40,42 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
sql("create table datamaptest (a string, b string, c string) stored by 'carbondata'")
}
-
- test("test datamap create") {
- sql("create datamap datamap1 on table datamaptest using 'new.class'")
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 1)
- assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap1"))
- assert(dataMapSchemaList.get(0).getClassName.equals("new.class"))
+ val newClass = "org.apache.spark.sql.CarbonSource"
+
+ test("test datamap create: don't support using class, only support short name") {
+ intercept[MalformedDataMapCommandException] {
+ sql(s"CREATE DATAMAP datamap1 ON TABLE datamaptest USING '$newClass'")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 1)
+ assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap1"))
+ assert(dataMapSchemaList.get(0).getClassName.equals(newClass))
+ }
}
- test("test datamap create with dmproperties") {
- sql("create datamap datamap2 on table datamaptest using 'new.class' dmproperties('key'='value')")
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
- assert(dataMapSchemaList.get(1).getDataMapName.equals("datamap2"))
- assert(dataMapSchemaList.get(1).getClassName.equals("new.class"))
- assert(dataMapSchemaList.get(1).getProperties.get("key").equals("value"))
+ test("test datamap create with dmproperties: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 2)
+ assert(dataMapSchemaList.get(1).getDataMapName.equals("datamap2"))
+ assert(dataMapSchemaList.get(1).getClassName.equals(newClass))
+ assert(dataMapSchemaList.get(1).getProperties.get("key").equals("value"))
+ }
}
- test("test datamap create with existing name") {
- intercept[Exception] {
+ test("test datamap create with existing name: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
sql(
- "create datamap datamap2 on table datamaptest using 'new.class' dmproperties('key'='value')")
+ s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 2)
}
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
}
test("test datamap create with preagg") {
@@ -79,10 +85,10 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
assert(table != null)
val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 3)
- assert(dataMapSchemaList.get(2).getDataMapName.equals("datamap3"))
- assert(dataMapSchemaList.get(2).getProperties.get("key").equals("value"))
- assert(dataMapSchemaList.get(2).getChildSchema.getTableName.equals("datamaptest_datamap3"))
+ assert(dataMapSchemaList.size() == 1)
+ assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap3"))
+ assert(dataMapSchemaList.get(0).getProperties.get("key").equals("value"))
+ assert(dataMapSchemaList.get(0).getChildSchema.getTableName.equals("datamaptest_datamap3"))
}
test("check hivemetastore after drop datamap") {
@@ -110,7 +116,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
}
}
- test("drop the table having pre-aggregate"){
+ test("drop the table having pre-aggregate") {
try {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
@@ -140,13 +146,24 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
test("test datamap create with preagg with duplicate name") {
intercept[Exception] {
sql(
- "create datamap datamap2 on table datamaptest using 'preaggregate' dmproperties('key'='value') as select count(a) from datamaptest")
-
+ s"""
+ | CREATE DATAMAP datamap2 ON TABLE datamaptest
+ | USING 'preaggregate'
+ | DMPROPERTIES('key'='value')
+ | AS SELECT COUNT(a) FROM datamaptest
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP datamap2 ON TABLE datamaptest
+ | USING 'preaggregate'
+ | DMPROPERTIES('key'='value')
+ | AS SELECT COUNT(a) FROM datamaptest
+ """.stripMargin)
}
val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
assert(table != null)
val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 3)
+ assert(dataMapSchemaList.size() == 2)
}
test("test datamap drop with preagg") {
@@ -157,25 +174,29 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
assert(table != null)
val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 3)
+ assert(dataMapSchemaList.size() == 2)
}
- test("test show datamap without preaggregate") {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'new.class' dmproperties('key'='value')")
- sql("create datamap datamap2 on table datamapshowtest using 'new.class' dmproperties('key'='value')")
- checkExistence(sql("show datamap on table datamapshowtest"), true, "datamap1", "datamap2", "(NA)", "new.class")
+ test("test show datamap without preaggregate: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql(s"CREATE DATAMAP datamap1 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
+ checkExistence(sql("SHOW DATAMAP ON TABLE datamapshowtest"), true, "datamap1", "datamap2", "(NA)", newClass)
+ }
}
- test("test show datamap with preaggregate") {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
- sql("create datamap datamap2 on table datamapshowtest using 'new.class' dmproperties('key'='value')")
- val frame = sql("show datamap on table datamapshowtest")
- assert(frame.collect().length == 2)
- checkExistence(frame, true, "datamap1", "datamap2", "(NA)", "new.class", "default.datamapshowtest_datamap1")
+ test("test show datamap with preaggregate: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
+ val frame = sql("show datamap on table datamapshowtest")
+ assert(frame.collect().length == 2)
+ checkExistence(frame, true, "datamap1", "datamap2", "(NA)", newClass, "default.datamapshowtest_datamap1")
+ }
}
test("test show datamap with no datamap") {
@@ -184,15 +205,17 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
assert(sql("show datamap on table datamapshowtest").collect().length == 0)
}
- test("test show datamap after dropping datamap") {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
- sql("create datamap datamap2 on table datamapshowtest using 'new.class' dmproperties('key'='value')")
- sql("drop datamap datamap1 on table datamapshowtest")
- val frame = sql("show datamap on table datamapshowtest")
- assert(frame.collect().length == 1)
- checkExistence(frame, true, "datamap2", "(NA)", "new.class")
+ test("test show datamap after dropping datamap: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
+ sql("drop datamap datamap1 on table datamapshowtest")
+ val frame = sql("show datamap on table datamapshowtest")
+ assert(frame.collect().length == 1)
+ checkExistence(frame, true, "datamap2", "(NA)", newClass)
+ }
}
test("test if preaggregate load is successfull for hivemetastore") {
@@ -217,7 +240,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
- checkAnswer(sql("select * from uniqdata_uniqdata_agg"),Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
+ checkAnswer(sql("select * from uniqdata_uniqdata_agg"), Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
sql("drop datamap if exists uniqdata_agg on table uniqdata")
}
@@ -229,16 +252,16 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
sql("insert into main select 10,11,'amy',12")
sql("insert into main select 10,11,'amy',14")
sql("create datamap preagg on table main " +
- "using 'preaggregate' " +
- s"dmproperties ('path'='$path') " +
- "as select name,avg(salary) from main group by name")
+ "using 'preaggregate' " +
+ s"dmproperties ('path'='$path') " +
+ "as select name,avg(salary) from main group by name")
assertResult(true)(new File(path).exists())
assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
- .list(new FilenameFilter {
- override def accept(dir: File, name: String): Boolean = {
- name.contains(CarbonCommonConstants.FACT_FILE_EXT)
- }
- }).length > 0)
+ .list(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.contains(CarbonCommonConstants.FACT_FILE_EXT)
+ }
+ }).length > 0)
checkAnswer(sql("select name,avg(salary) from main group by name"), Row("amy", 13.0))
checkAnswer(sql("select * from main_preagg"), Row("amy", 26, 2))
sql("drop datamap preagg on table main")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java
new file mode 100644
index 0000000..a05d8e6
--- /dev/null
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.exception;
+
+/**
+ * Throw exception when using illegal argument
+ */
+public class MalformedDataMapCommandException extends MalformedCarbonCommandException {
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ public MalformedDataMapCommandException(String msg) {
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 8e00635..c4d32b4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -16,18 +16,15 @@
*/
package org.apache.spark.sql.execution.command.datamap
-import scala.collection.JavaConverters._
-
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.command.preaaggregate.{CreatePreAggregateTableCommand, PreAggregateUtil}
+import org.apache.spark.sql.execution.command.preaaggregate.CreatePreAggregateTableCommand
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider._
+import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, MalformedDataMapCommandException}
/**
* Below command class will be used to create datamap on table
@@ -41,70 +38,63 @@ case class CarbonCreateDataMapCommand(
queryString: Option[String])
extends AtomicRunnableCommand {
- var createPreAggregateTableCommands: Seq[CreatePreAggregateTableCommand] = _
+ var createPreAggregateTableCommands: CreatePreAggregateTableCommand = _
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
// since streaming segment does not support building index and pre-aggregate yet,
// so streaming table does not support create datamap
val carbonTable =
- CarbonEnv.getCarbonTable(tableIdentifier.database, tableIdentifier.table)(sparkSession)
+ CarbonEnv.getCarbonTable(tableIdentifier.database, tableIdentifier.table)(sparkSession)
if (carbonTable.isStreamingTable) {
throw new MalformedCarbonCommandException("Streaming table does not support creating datamap")
}
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- if (dmClassName.equals("org.apache.carbondata.datamap.AggregateDataMapHandler") ||
- dmClassName.equalsIgnoreCase("preaggregate")) {
- val timeHierarchyString = dmproperties.get(CarbonCommonConstants.TIMESERIES_HIERARCHY)
- createPreAggregateTableCommands = if (timeHierarchyString.isDefined) {
+
+ if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
+ dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ TimeSeriesUtil.validateTimeSeriesGranularity(dmproperties, dmClassName)
+ createPreAggregateTableCommands = if (dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
val details = TimeSeriesUtil
- .validateAndGetTimeSeriesHierarchyDetails(
- timeHierarchyString.get)
- val updatedDmProperties = dmproperties - CarbonCommonConstants.TIMESERIES_HIERARCHY
- details.map { f =>
- CreatePreAggregateTableCommand(dataMapName + '_' + f._1,
- tableIdentifier,
- dmClassName,
- updatedDmProperties,
- queryString.get,
- Some(f._1))
- }.toSeq
+ .getTimeSeriesGranularityDetails(dmproperties, dmClassName)
+ val updatedDmProperties = dmproperties - details._1
+ CreatePreAggregateTableCommand(dataMapName,
+ tableIdentifier,
+ dmClassName,
+ updatedDmProperties,
+ queryString.get,
+ Some(details._1))
} else {
- Seq(CreatePreAggregateTableCommand(
+ CreatePreAggregateTableCommand(
dataMapName,
tableIdentifier,
dmClassName,
dmproperties,
queryString.get
- ))
+ )
}
- createPreAggregateTableCommands.flatMap(_.processMetadata(sparkSession))
+ createPreAggregateTableCommands.processMetadata(sparkSession)
} else {
- val dataMapSchema = new DataMapSchema(dataMapName, dmClassName)
- dataMapSchema.setProperties(new java.util.HashMap[String, String](dmproperties.asJava))
- val dbName = CarbonEnv.getDatabaseName(tableIdentifier.database)(sparkSession)
- // upadting the parent table about dataschema
- PreAggregateUtil.updateMainTable(dbName, tableIdentifier.table, dataMapSchema, sparkSession)
+ throw new MalformedDataMapCommandException("Unknown data map type " + dmClassName)
}
- LOGGER.audit(s"DataMap $dataMapName successfully added to Table ${ tableIdentifier.table }")
+ LOGGER.audit(s"DataMap $dataMapName successfully added to Table ${tableIdentifier.table}")
Seq.empty
}
override def processData(sparkSession: SparkSession): Seq[Row] = {
- if (dmClassName.equals("org.apache.carbondata.datamap.AggregateDataMapHandler") ||
- dmClassName.equalsIgnoreCase("preaggregate")) {
- createPreAggregateTableCommands.flatMap(_.processData(sparkSession))
+ if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
+ dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ createPreAggregateTableCommands.processData(sparkSession)
} else {
- Seq.empty
+ throw new MalformedDataMapCommandException("Unknown data map type " + dmClassName)
}
}
override def undoMetadata(sparkSession: SparkSession, exception: Exception): Seq[Row] = {
- if (dmClassName.equals("org.apache.carbondata.datamap.AggregateDataMapHandler") ||
- dmClassName.equalsIgnoreCase("preaggregate")) {
- val timeHierarchyString = dmproperties.get(CarbonCommonConstants.TIMESERIES_HIERARCHY)
- createPreAggregateTableCommands.flatMap(_.undoMetadata(sparkSession, exception))
+ if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
+ dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ createPreAggregateTableCommands.undoMetadata(sparkSession, exception)
} else {
- Seq.empty
+ throw new MalformedDataMapCommandException("Unknown data map type " + dmClassName)
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index a75a06f..dbbf90c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -104,9 +104,9 @@ case class CreatePreAggregateTableCommand(
TimeSeriesUtil.validateTimeSeriesEventTime(dmProperties, parentTable)
TimeSeriesUtil.validateEventTimeColumnExitsInSelect(
fieldRelationMap,
- dmProperties.get(CarbonCommonConstants.TIMESERIES_EVENTTIME).get)
+ dmProperties.get(TimeSeriesUtil.TIMESERIES_EVENTTIME).get)
TimeSeriesUtil.updateTimeColumnSelect(fieldRelationMap,
- dmProperties.get(CarbonCommonConstants.TIMESERIES_EVENTTIME).get,
+ dmProperties.get(TimeSeriesUtil.TIMESERIES_EVENTTIME).get,
timeSeriesFunction.get)
}
tableModel.parentTable = Some(parentTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
index 4fe9df0..987d4fe 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
@@ -18,32 +18,33 @@ package org.apache.spark.sql.execution.command.timeseries
import org.apache.spark.sql.execution.command.{DataMapField, Field}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.core.metadata.schema.datamap.Granularity
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.preagg.TimeSeriesUDF
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, MalformedDataMapCommandException}
/**
* Utility class for time series to keep
*/
object TimeSeriesUtil {
+ val TIMESERIES_EVENTTIME = "event_time"
+
/**
* Below method will be used to validate whether column mentioned in time series
* is timestamp column or not
*
- * @param dmproperties
- * data map properties
- * @param parentTable
- * parent table
+ * @param dmproperties data map properties
+ * @param parentTable parent table
* @return whether time stamp column
*/
def validateTimeSeriesEventTime(dmproperties: Map[String, String],
parentTable: CarbonTable) {
- val eventTime = dmproperties.get(CarbonCommonConstants.TIMESERIES_EVENTTIME)
+ val eventTime = dmproperties.get(TIMESERIES_EVENTTIME)
if (!eventTime.isDefined) {
- throw new MalformedCarbonCommandException("Eventtime not defined in time series")
+ throw new MalformedCarbonCommandException("event_time not defined in time series")
} else {
val carbonColumn = parentTable.getColumnByName(parentTable.getTableName, eventTime.get)
if (carbonColumn.getDataType != DataTypes.TIMESTAMP) {
@@ -55,13 +56,79 @@ object TimeSeriesUtil {
}
/**
+ * validate TimeSeries Granularity
+ *
+ * @param dmProperties datamap properties
+ * @param dmClassName datamap class name
+ * @return whether find only one granularity
+ */
+ def validateTimeSeriesGranularity(
+ dmProperties: Map[String, String],
+ dmClassName: String): Boolean = {
+ var isFound = false
+
+ // 1. granularity only support one
+ for (granularity <- Granularity.values()) {
+ if (dmProperties.get(granularity.getName).isDefined) {
+ if (isFound) {
+ throw new MalformedDataMapCommandException(
+ s"Only one granularity level can be defined")
+ } else {
+ isFound = true
+ }
+ }
+ }
+
+ // 2. check whether timeseries and granularity match
+ if (isFound && !dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ throw new MalformedDataMapCommandException(
+ s"${TIMESERIES.toString} keyword missing")
+ } else if (!isFound && dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ throw new MalformedDataMapCommandException(
+ s"${TIMESERIES.toString} should define time granularity")
+ } else if (isFound) {
+ true
+ } else {
+ false
+ }
+ }
+
+ /**
+ * get TimeSeries Granularity key and value
+ * check the value
+ *
+ * TODO:we will support value not only equal to 1 in the future
+ *
+ * @param dmProperties datamap properties
+ * @param dmClassName datamap class name
+ * @return key and value tuple
+ */
+ def getTimeSeriesGranularityDetails(
+ dmProperties: Map[String, String],
+ dmClassName: String): (String, String) = {
+
+ val defaultValue = "1"
+ for (granularity <- Granularity.values()) {
+ if (dmProperties.get(granularity.getName).isDefined &&
+ dmProperties.get(granularity.getName).get.equalsIgnoreCase(defaultValue)) {
+ return (granularity.toString.toLowerCase, dmProperties.get(granularity.getName).get)
+ }
+ }
+
+ throw new MalformedDataMapCommandException(
+ s"Granularity only support $defaultValue")
+ }
+
+ /**
* Below method will be used to validate the hierarchy of time series and its value
* validation will be done whether hierarchy order is proper or not and hierarchy level
* value
+ * TODO: we should remove this method
*
* @param timeSeriesHierarchyDetails
* time series hierarchy string
*/
+ @deprecated
def validateAndGetTimeSeriesHierarchyDetails(timeSeriesHierarchyDetails: String): Array[
(String, String)] = {
val updatedtimeSeriesHierarchyDetails = timeSeriesHierarchyDetails.toLowerCase
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
index 0f934cb..eb52910 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
@@ -260,4 +260,12 @@ class CarbonDataSourceSuite extends Spark2QueryTest with BeforeAndAfterAll {
sql("drop table if exists carbon_test")
assert(exception.contains("Table creation failed. Table name cannot contain blank space"))
}
+
+ test("test create table: using") {
+ sql("DROP TABLE IF EXISTS usingTable")
+ val e: Exception = intercept[ClassNotFoundException] {
+ sql("CREATE TABLE usingTable(name STRING) USING abc")
+ }
+ assert(e.getMessage.contains("Failed to find data source: abc"))
+ }
}
[13/50] [abbrv] carbondata git commit: [CARBONDATA-1988] Fixed bug to
remove empty partition directory for drop partition command
Posted by ra...@apache.org.
[CARBONDATA-1988] Fixed bug to remove empty partition directory for drop partition command
This closes #1786
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3c3f33df
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3c3f33df
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3c3f33df
Branch: refs/heads/fgdatamap
Commit: 3c3f33dfcae84af11054eab8bde9ea83f1cf9f0d
Parents: e349820
Author: Geetika Gupta <ge...@knoldus.in>
Authored: Wed Jan 10 16:23:55 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Jan 31 12:06:14 2018 +0530
----------------------------------------------------------------------
.../core/metadata/PartitionMapFileStore.java | 7 ++++-
.../StandardPartitionTableDropTestCase.scala | 27 ++++++++++++++++++++
.../spark/rdd/CarbonDropPartitionRDD.scala | 6 +++--
.../management/CarbonLoadDataCommand.scala | 9 ++++---
...rbonAlterTableDropHivePartitionCommand.scala | 6 +++--
5 files changed, 47 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c3f33df/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index 355d083..1e9cbc4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -313,9 +313,12 @@ public class PartitionMapFileStore {
* @param uniqueId
* @param success
*/
- public void commitPartitions(String segmentPath, final String uniqueId, boolean success) {
+ public void commitPartitions(String segmentPath, final String uniqueId, boolean success,
+ String tablePath, List<String> partitionsToDrop) {
CarbonFile carbonFile = FileFactory
.getCarbonFile(segmentPath + "/" + uniqueId + CarbonTablePath.PARTITION_MAP_EXT + ".tmp");
+ CarbonFile carbonPartFile = FileFactory
+ .getCarbonFile(tablePath + "/" + partitionsToDrop.get(0));
// write partition info to new file.
if (carbonFile.exists()) {
if (success) {
@@ -324,6 +327,8 @@ public class PartitionMapFileStore {
carbonFile.delete();
}
}
+ //Remove the partition directory from table path
+ carbonPartFile.delete();
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c3f33df/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
index 2aa9145..aac823a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
@@ -16,7 +16,10 @@
*/
package org.apache.carbondata.spark.testsuite.standardpartition
+import java.nio.file.{Files, LinkOption, Paths}
+
import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.TestQueryExecutor
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
@@ -182,6 +185,29 @@ class StandardPartitionTableDropTestCase extends QueryTest with BeforeAndAfterAl
Seq(Row(0)))
}
+ test("test dropping on partition table for int partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionone1 (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ checkAnswer(
+ sql(s"""select count (*) from partitionone1"""),
+ sql(s"""select count (*) from originTable"""))
+
+ checkAnswer(
+ sql(s"""select count (*) from partitionone1 where empno=11"""),
+ sql(s"""select count (*) from originTable where empno=11"""))
+ sql(s"""ALTER TABLE partitionone1 DROP PARTITION(empno='11')""")
+ assert(Files.notExists(Paths.get(TestQueryExecutor.warehouse + "/partitionone1/" + "empno=11"), LinkOption.NOFOLLOW_LINKS))
+ sql("drop table if exists partitionone1")
+ }
+
override def afterAll = {
dropTable
}
@@ -195,6 +221,7 @@ class StandardPartitionTableDropTestCase extends QueryTest with BeforeAndAfterAl
sql("drop table if exists partitionshow")
sql("drop table if exists staticpartition")
sql("drop table if exists partitionallcompaction")
+ sql("drop table if exists partitionone1")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c3f33df/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
index 0a79295..4806f9f 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
@@ -103,7 +103,8 @@ class CarbonDropPartitionCommitRDD(
tablePath: String,
segments: Seq[String],
success: Boolean,
- uniqueId: String)
+ uniqueId: String,
+ partitions: Seq[String])
extends CarbonRDD[String](sc, Nil) {
override def getPartitions: Array[Partition] = {
@@ -117,7 +118,8 @@ class CarbonDropPartitionCommitRDD(
val split = theSplit.asInstanceOf[CarbonDropPartition]
logInfo("Commit partition information from : " + split.segmentPath)
- new PartitionMapFileStore().commitPartitions(split.segmentPath, uniqueId, success)
+ new PartitionMapFileStore().commitPartitions(split.segmentPath, uniqueId, success, tablePath,
+ partitions.toList.asJava)
var havePair = false
var finished = false
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c3f33df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 7afbd92..226a625 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -813,7 +813,8 @@ case class CarbonLoadDataCommand(
table.getTablePath,
segments.asScala,
success = false,
- uniqueId).collect()
+ uniqueId,
+ partitionNames.toSeq).collect()
throw e
}
@@ -827,7 +828,8 @@ case class CarbonLoadDataCommand(
table.getTablePath,
segments.asScala,
success = false,
- uniqueId).collect()
+ uniqueId,
+ partitionNames.toSeq).collect()
throw e
}
// Commit the removed partitions in carbon store.
@@ -836,7 +838,8 @@ case class CarbonLoadDataCommand(
table.getTablePath,
segments.asScala,
success = true,
- uniqueId).collect()
+ uniqueId,
+ partitionNames.toSeq).collect()
// get valid segments
val validsegments =
new SegmentStatusManager(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c3f33df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
index dbd686b..c3509a3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
@@ -129,7 +129,8 @@ case class CarbonAlterTableDropHivePartitionCommand(
table.getTablePath,
segments.asScala,
false,
- uniqueId).collect()
+ uniqueId,
+ partitionNames.toSeq).collect()
throw e
}
// commit the drop partitions from carbon store
@@ -137,7 +138,8 @@ case class CarbonAlterTableDropHivePartitionCommand(
table.getTablePath,
segments.asScala,
true,
- uniqueId).collect()
+ uniqueId,
+ partitionNames.toSeq).collect()
// Update the loadstatus with update time to clear cache from driver.
val segmentSet = new util.HashSet[String](new SegmentStatusManager(table
.getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments)
[40/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
index 8c8d08f..a689d8e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
@@ -124,22 +124,22 @@ public class UnsafeFixedLengthDimensionDataChunkStore
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// based on index we need to calculate the actual position in memory block
- index = index * columnValueSize;
+ rowId = rowId * columnValueSize;
int compareResult = 0;
for (int i = 0; i < compareValue.length; i++) {
compareResult = (CarbonUnsafe.getUnsafe()
- .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index)
+ .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + rowId)
& 0xff) - (compareValue[i] & 0xff);
if (compareResult != 0) {
break;
}
- index++;
+ rowId++;
}
return compareResult;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index 36b2bd8..e1eb378 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -189,11 +189,11 @@ public class UnsafeVariableLengthDimesionDataChunkStore
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// now to get the row from memory block we need to do following thing
// 1. first get the current offset
// 2. if it's not a last row- get the next row offset
@@ -201,13 +201,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// else subtract the current row offset
// with complete data length get the offset of set of data
int currentDataOffset = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((long)index
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((long) rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE * 1L));
short length = 0;
// calculating the length of data
- if (index < numberOfRows - 1) {
+ if (rowId < numberOfRows - 1) {
int OffsetOfNextdata = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1)
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1)
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
length = (short) (OffsetOfNextdata - (currentDataOffset
+ CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
index 74d268a..e2a4161 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
@@ -29,13 +29,6 @@ public class ColumnGroupModel {
private int noOfColumnsStore;
/**
- * whether given index is columnar or not
- * true: columnar
- * false: row block
- */
- private boolean[] columnarStore;
-
- /**
* column groups
* e.g
* {{0,1,2},3,4,{5,6}}
@@ -77,15 +70,6 @@ public class ColumnGroupModel {
}
/**
- * it's an identifier for row block or single column block
- *
- * @param columnarStore
- */
- public void setColumnarStore(boolean[] columnarStore) {
- this.columnarStore = columnarStore;
- }
-
- /**
* set column groups
*
* @param columnGroups
@@ -95,16 +79,6 @@ public class ColumnGroupModel {
}
/**
- * check if given column group is columnar
- *
- * @param colGroup
- * @return true if given block is columnar
- */
- public boolean isColumnar(int colGroup) {
- return columnarStore[colGroup];
- }
-
- /**
* @return columngroups
*/
public int[][] getColumnGroup() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
deleted file mode 100644
index 182c8eb..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.impl;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.FileHolder;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public class DFSFileHolderImpl implements FileHolder {
- /**
- * cache to hold filename and its stream
- */
- private Map<String, FSDataInputStream> fileNameAndStreamCache;
-
- private String queryId;
-
- private boolean readPageByPage;
-
-
- public DFSFileHolderImpl() {
- this.fileNameAndStreamCache =
- new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- @Override public byte[] readByteArray(String filePath, long offset, int length)
- throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return read(fileChannel, length, offset);
- }
-
- /**
- * This method will be used to check whether stream is already present in
- * cache or not for filepath if not present then create it and then add to
- * cache, other wise get from cache
- *
- * @param filePath fully qualified file path
- * @return channel
- */
- public FSDataInputStream updateCache(String filePath) throws IOException {
- FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
- if (null == fileChannel) {
- Path pt = new Path(filePath);
- FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
- fileChannel = fs.open(pt);
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- return fileChannel;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and position
- *
- * @param channel file channel
- * @param size number of bytes
- * @param offset position
- * @return byte buffer
- */
- private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException {
- byte[] byteBffer = new byte[size];
- channel.seek(offset);
- channel.readFully(byteBffer);
- return byteBffer;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and position
- *
- * @param channel file channel
- * @param size number of bytes
- * @return byte buffer
- */
- private byte[] read(FSDataInputStream channel, int size) throws IOException {
- byte[] byteBffer = new byte[size];
- channel.readFully(byteBffer);
- return byteBffer;
- }
-
- @Override public int readInt(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readInt();
- }
-
- @Override public long readDouble(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readLong();
- }
-
- @Override public void finish() throws IOException {
- for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) {
- FSDataInputStream channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- }
- }
-
- @Override public byte[] readByteArray(String filePath, int length) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return read(fileChannel, length);
- }
-
- @Override public long readLong(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readLong();
- }
-
- @Override public int readInt(String filePath) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return fileChannel.readInt();
- }
-
- @Override public ByteBuffer readByteBuffer(String filePath, long offset, int length)
- throws IOException {
- byte[] readByteArray = readByteArray(filePath, offset, length);
- ByteBuffer byteBuffer = ByteBuffer.wrap(readByteArray);
- byteBuffer.rewind();
- return byteBuffer;
- }
-
- @Override public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- @Override public String getQueryId() {
- return queryId;
- }
-
- @Override public void setReadPageByPage(boolean isReadPageByPage) {
- this.readPageByPage = isReadPageByPage;
- }
-
- @Override public boolean isReadPageByPage() {
- return readPageByPage;
- }
-
- public Map<String, FSDataInputStream> getFileNameAndStreamCache() {
- return fileNameAndStreamCache;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileReaderImpl.java
new file mode 100644
index 0000000..1a0cd41
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileReaderImpl.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.impl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.FileReader;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class DFSFileReaderImpl implements FileReader {
+ /**
+ * cache to hold filename and its stream
+ */
+ private Map<String, FSDataInputStream> fileNameAndStreamCache;
+
+ private boolean readPageByPage;
+
+ public DFSFileReaderImpl() {
+ this.fileNameAndStreamCache =
+ new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ }
+
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return read(fileChannel, length, offset);
+ }
+
+ /**
+ * This method will be used to check whether stream is already present in
+ * cache or not for filepath if not present then create it and then add to
+ * cache, other wise get from cache
+ *
+ * @param filePath fully qualified file path
+ * @return channel
+ */
+ private FSDataInputStream updateCache(String filePath) throws IOException {
+ FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
+ if (null == fileChannel) {
+ Path pt = new Path(filePath);
+ FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
+ fileChannel = fs.open(pt);
+ fileNameAndStreamCache.put(filePath, fileChannel);
+ }
+ return fileChannel;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and position
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @param offset position
+ * @return byte buffer
+ */
+ private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException {
+ byte[] byteBffer = new byte[size];
+ channel.seek(offset);
+ channel.readFully(byteBffer);
+ return byteBffer;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and position
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @return byte buffer
+ */
+ private byte[] read(FSDataInputStream channel, int size) throws IOException {
+ byte[] byteBffer = new byte[size];
+ channel.readFully(byteBffer);
+ return byteBffer;
+ }
+
+ @Override public int readInt(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readInt();
+ }
+
+ @Override public long readDouble(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
+ }
+
+ @Override public void finish() throws IOException {
+ for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) {
+ FSDataInputStream channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
+ }
+ }
+ }
+
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return read(fileChannel, length);
+ }
+
+ @Override public long readLong(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
+ }
+
+ @Override public int readInt(String filePath) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return fileChannel.readInt();
+ }
+
+ @Override public ByteBuffer readByteBuffer(String filePath, long offset, int length)
+ throws IOException {
+ byte[] readByteArray = readByteArray(filePath, offset, length);
+ ByteBuffer byteBuffer = ByteBuffer.wrap(readByteArray);
+ byteBuffer.rewind();
+ return byteBuffer;
+ }
+
+ @Override public void setReadPageByPage(boolean isReadPageByPage) {
+ this.readPageByPage = isReadPageByPage;
+ }
+
+ @Override public boolean isReadPageByPage() {
+ return readPageByPage;
+ }
+
+ public Map<String, FSDataInputStream> getFileNameAndStreamCache() {
+ return fileNameAndStreamCache;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java
index 67648fe..b58a473 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java
@@ -17,24 +17,28 @@
package org.apache.carbondata.core.datastore.impl;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.filesystem.*;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.filesystem.AlluxioCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.ViewFSCarbonFile;
import org.apache.hadoop.conf.Configuration;
public class DefaultFileTypeProvider implements FileTypeInerface {
- public FileHolder getFileHolder(FileFactory.FileType fileType) {
+ public FileReader getFileHolder(FileFactory.FileType fileType) {
switch (fileType) {
case LOCAL:
- return new FileHolderImpl();
+ return new FileReaderImpl();
case HDFS:
case ALLUXIO:
case VIEWFS:
case S3:
- return new DFSFileHolderImpl();
+ return new DFSFileReaderImpl();
default:
- return new FileHolderImpl();
+ return new FileReaderImpl();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index daf6d93..29c692f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -28,7 +28,7 @@ import java.nio.channels.FileChannel;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.commons.io.FileUtils;
@@ -62,7 +62,7 @@ public final class FileFactory {
return configuration;
}
- public static FileHolder getFileHolder(FileType fileType) {
+ public static FileReader getFileHolder(FileType fileType) {
return fileFileTypeInerface.getFileHolder(fileType);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
deleted file mode 100644
index cc589b7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.impl;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.FileHolder;
-
-public class FileHolderImpl implements FileHolder {
- /**
- * cache to hold filename and its stream
- */
- private Map<String, FileChannel> fileNameAndStreamCache;
- private String queryId;
-
- private boolean readPageByPage;
-
- /**
- * FileHolderImpl Constructor
- * It will create the cache
- */
- public FileHolderImpl() {
- this.fileNameAndStreamCache =
- new HashMap<String, FileChannel>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- public FileHolderImpl(int capacity) {
- this.fileNameAndStreamCache = new HashMap<String, FileChannel>(capacity);
- }
-
- /**
- * This method will be used to read the byte array from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return read byte array
- */
- @Override public byte[] readByteArray(String filePath, long offset, int length)
- throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, length, offset);
- return byteBffer.array();
- }
-
- /**
- * This method will be used to close all the streams currently present in the cache
- */
- @Override public void finish() throws IOException {
- for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
- FileChannel channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- }
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- @Override public int readInt(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
- return byteBffer.getInt();
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @return read int
- */
- @Override public int readInt(String filePath) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE);
- return byteBffer.getInt();
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- @Override public long readDouble(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
- return byteBffer.getLong();
- }
-
- /**
- * This method will be used to check whether stream is already present in
- * cache or not for filepath if not present then create it and then add to
- * cache, other wise get from cache
- *
- * @param filePath fully qualified file path
- * @return channel
- */
- private FileChannel updateCache(String filePath) throws FileNotFoundException {
- FileChannel fileChannel = fileNameAndStreamCache.get(filePath);
- if (null == fileChannel) {
- FileInputStream stream = new FileInputStream(filePath);
- fileChannel = stream.getChannel();
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- return fileChannel;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and position
- *
- * @param channel file channel
- * @param size number of bytes
- * @param offset position
- * @return byte buffer
- */
- private ByteBuffer read(FileChannel channel, int size, long offset) throws IOException {
- ByteBuffer byteBffer = ByteBuffer.allocate(size);
- channel.position(offset);
- channel.read(byteBffer);
- byteBffer.rewind();
- return byteBffer;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and position
- *
- * @param channel file channel
- * @param size number of bytes
- * @return byte buffer
- */
- private ByteBuffer read(FileChannel channel, int size) throws IOException {
- ByteBuffer byteBffer = ByteBuffer.allocate(size);
- channel.read(byteBffer);
- byteBffer.rewind();
- return byteBffer;
- }
-
-
- /**
- * This method will be used to read the byte array from file based on length(number of bytes)
- *
- * @param filePath fully qualified file path
- * @param length number of bytes to be read
- * @return read byte array
- */
- @Override public byte[] readByteArray(String filePath, int length) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, length);
- return byteBffer.array();
- }
-
- /**
- * This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- @Override public long readLong(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
- return byteBffer.getLong();
- }
-
- @Override public ByteBuffer readByteBuffer(String filePath, long offset, int length)
- throws IOException {
- ByteBuffer byteBuffer = ByteBuffer.allocate(length);
- FileChannel fileChannel = updateCache(filePath);
- fileChannel.position(offset);
- fileChannel.read(byteBuffer);
- byteBuffer.rewind();
- return byteBuffer;
- }
-
- @Override public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- @Override public String getQueryId() {
- return queryId;
- }
-
- @Override public void setReadPageByPage(boolean isReadPageByPage) {
- this.readPageByPage = isReadPageByPage;
- }
-
- @Override public boolean isReadPageByPage() {
- return readPageByPage;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
new file mode 100644
index 0000000..6fef278
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.impl;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.FileReader;
+
+public class FileReaderImpl implements FileReader {
+ /**
+ * cache to hold filename and its stream
+ */
+ private Map<String, FileChannel> fileNameAndStreamCache;
+
+ private boolean readPageByPage;
+
+ /**
+ * FileReaderImpl Constructor
+ * It will create the cache
+ */
+ public FileReaderImpl() {
+ this.fileNameAndStreamCache =
+ new HashMap<String, FileChannel>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ }
+
+ public FileReaderImpl(int capacity) {
+ this.fileNameAndStreamCache = new HashMap<String, FileChannel>(capacity);
+ }
+
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, length, offset);
+ return byteBffer.array();
+ }
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ @Override public void finish() throws IOException {
+ for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
+ FileChannel channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
+ }
+ }
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ @Override public int readInt(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
+ return byteBffer.getInt();
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ @Override public int readInt(String filePath) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ return byteBffer.getInt();
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ @Override public long readDouble(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
+ return byteBffer.getLong();
+ }
+
+ /**
+ * This method will be used to check whether stream is already present in
+ * cache or not for filepath if not present then create it and then add to
+ * cache, other wise get from cache
+ *
+ * @param filePath fully qualified file path
+ * @return channel
+ */
+ private FileChannel updateCache(String filePath) throws FileNotFoundException {
+ FileChannel fileChannel = fileNameAndStreamCache.get(filePath);
+ if (null == fileChannel) {
+ FileInputStream stream = new FileInputStream(filePath);
+ fileChannel = stream.getChannel();
+ fileNameAndStreamCache.put(filePath, fileChannel);
+ }
+ return fileChannel;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and position
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @param offset position
+ * @return byte buffer
+ */
+ private ByteBuffer read(FileChannel channel, int size, long offset) throws IOException {
+ ByteBuffer byteBffer = ByteBuffer.allocate(size);
+ channel.position(offset);
+ channel.read(byteBffer);
+ byteBffer.rewind();
+ return byteBffer;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and position
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @return byte buffer
+ */
+ private ByteBuffer read(FileChannel channel, int size) throws IOException {
+ ByteBuffer byteBffer = ByteBuffer.allocate(size);
+ channel.read(byteBffer);
+ byteBffer.rewind();
+ return byteBffer;
+ }
+
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, length);
+ return byteBffer.array();
+ }
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ @Override public long readLong(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
+ return byteBffer.getLong();
+ }
+
+ @Override public ByteBuffer readByteBuffer(String filePath, long offset, int length)
+ throws IOException {
+ ByteBuffer byteBuffer = ByteBuffer.allocate(length);
+ FileChannel fileChannel = updateCache(filePath);
+ fileChannel.position(offset);
+ fileChannel.read(byteBuffer);
+ byteBuffer.rewind();
+ return byteBuffer;
+ }
+
+ @Override public void setReadPageByPage(boolean isReadPageByPage) {
+ this.readPageByPage = isReadPageByPage;
+ }
+
+ @Override public boolean isReadPageByPage() {
+ return readPageByPage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java
index 4676278..413261c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java
@@ -17,14 +17,14 @@
package org.apache.carbondata.core.datastore.impl;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.hadoop.conf.Configuration;
public interface FileTypeInerface {
- FileHolder getFileHolder(FileFactory.FileType fileType);
+ FileReader getFileHolder(FileFactory.FileType fileType);
CarbonFile getCarbonFile(String path, FileFactory.FileType fileType);
CarbonFile getCarbonFile(String path, FileFactory.FileType fileType, Configuration configuration);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
index 19b1f1c..fe4cf83 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -18,9 +18,8 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.io.IOException;
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
@@ -31,38 +30,31 @@ import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
public abstract class AbstractBTreeLeafNode implements BTreeNode {
/**
- * Below method will be used to load the data block
- *
- * @param blockInfo block detail
- */
- protected BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache;
-
- /**
* number of keys in a btree
*/
- protected int numberOfKeys;
+ int numberOfKeys;
/**
* node number
*/
- protected long nodeNumber;
+ long nodeNumber;
/**
* Next node of the leaf
*/
- protected BTreeNode nextNode;
+ private BTreeNode nextNode;
/**
* max key of the column this will be used to check whether this leaf will
* be used for scanning or not
*/
- protected byte[][] maxKeyOfColumns;
+ byte[][] maxKeyOfColumns;
/**
* min key of the column this will be used to check whether this leaf will
* be used for scanning or not
*/
- protected byte[][] minKeyOfColumns;
+ byte[][] minKeyOfColumns;
/**
* Method to get the next block this can be used while scanning when
@@ -70,7 +62,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
*
* @return next block
*/
- @Override public int nodeSize() {
+ @Override public int numRows() {
return this.numberOfKeys;
}
@@ -109,7 +101,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
*
* @return block number
*/
- @Override public long nodeNumber() {
+ @Override public long nodeIndex() {
return nodeNumber;
}
@@ -174,11 +166,11 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* Below method will be used to get the dimension chunks
*
* @param fileReader file reader to read the chunks from file
- * @param blockIndexes indexes of the blocks need to be read
+ * @param columnIndexRange indexes of the blocks need to be read
* @return dimension data chunks
*/
- @Override public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
// No required here as leaf which will will be use this class will implement its own get
// dimension chunks
return null;
@@ -188,11 +180,11 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* Below method will be used to get the dimension chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndex block index to be read
+ * @param columnIndex block index to be read
* @return dimension data chunk
*/
- @Override public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader,
- int blockIndex) throws IOException {
+ @Override public DimensionRawColumnChunk readDimensionChunk(FileReader fileReader,
+ int columnIndex) throws IOException {
// No required here as leaf which will will be use this class will implement
// its own get dimension chunks
return null;
@@ -202,11 +194,11 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* Below method will be used to get the measure chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndexes block indexes to be read from file
+ * @param columnIndexRange block indexes to be read from file
* @return measure column data chunk
*/
- @Override public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
// No required here as leaf which will will be use this class will implement its own get
// measure chunks
return null;
@@ -216,30 +208,16 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* Below method will be used to read the measure chunk
*
* @param fileReader file read to read the file chunk
- * @param blockIndex block index to be read from file
+ * @param columnIndex block index to be read from file
* @return measure data chunk
*/
- @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
+ @Override public MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException {
// No required here as leaf which will will be use this class will implement its own get
// measure chunks
return null;
}
- /**
- * @param deleteDeltaDataCache
- */
- public void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache) {
-
- this.deleteDeltaDataCache = deleteDeltaDataCache;
- }
- /**
- * @return the segmentProperties
- */
- public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
- return deleteDeltaDataCache;
- }
-
@Override
public int getPageRowCount(int pageNumber) {
throw new UnsupportedOperationException("Unsupported operation");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
index 2f8aadf..688d56a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -106,7 +106,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
private BTreeNode findFirstLeafNode(IndexKey key, BTreeNode node) {
int childNodeIndex;
int low = 0;
- int high = node.nodeSize() - 1;
+ int high = node.numRows() - 1;
int mid = 0;
int compareRes = -1;
IndexKey[] nodeKeys = node.getNodeKeys();
@@ -156,7 +156,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
private BTreeNode findLastLeafNode(IndexKey key, BTreeNode node) {
int childNodeIndex;
int low = 0;
- int high = node.nodeSize() - 1;
+ int high = node.numRows() - 1;
int mid = 0;
int compareRes = -1;
IndexKey[] nodeKeys = node.getNodeKeys();
@@ -172,7 +172,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
} else {
int currentPos = mid;
// if key is matched then get the first entry
- while (currentPos + 1 < node.nodeSize()
+ while (currentPos + 1 < node.numRows()
&& compareIndexes(key, nodeKeys[currentPos + 1]) == 0) {
currentPos++;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
index ccc5e12..c200f8d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
@@ -19,10 +19,9 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
@@ -34,13 +33,6 @@ import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
public class BTreeNonLeafNode implements BTreeNode {
/**
- * Below method will be used to load the data block
- *
- * @param blockInfo block detail
- */
- protected BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache;
-
- /**
* Child nodes
*/
private BTreeNode[] children;
@@ -50,7 +42,7 @@ public class BTreeNonLeafNode implements BTreeNode {
*/
private List<IndexKey> listOfKeys;
- public BTreeNonLeafNode() {
+ BTreeNonLeafNode() {
// creating a list which will store all the indexes
listOfKeys = new ArrayList<IndexKey>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
@@ -120,7 +112,7 @@ public class BTreeNonLeafNode implements BTreeNode {
*
* @return number of keys in the block
*/
- @Override public int nodeSize() {
+ @Override public int numRows() {
return listOfKeys.size();
}
@@ -131,11 +123,11 @@ public class BTreeNonLeafNode implements BTreeNode {
*
* @return block number
*/
- @Override public long nodeNumber() {
+ @Override public long nodeIndex() {
throw new UnsupportedOperationException("Unsupported operation");
}
- @Override public String blockletId() {
+ @Override public short blockletIndex() {
throw new UnsupportedOperationException("Unsupported operation");
}
@@ -171,11 +163,11 @@ public class BTreeNonLeafNode implements BTreeNode {
* Below method will be used to get the dimension chunks
*
* @param fileReader file reader to read the chunks from file
- * @param blockIndexes indexes of the blocks need to be read
+ * @param columnIndexRange indexes of the blocks need to be read
* @return dimension data chunks
*/
- @Override public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader,
- int[][] blockIndexes) {
+ @Override public DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader,
+ int[][] columnIndexRange) {
// operation of getting the dimension chunks is not supported as its a
// non leaf node
@@ -191,8 +183,8 @@ public class BTreeNonLeafNode implements BTreeNode {
* @param fileReader file reader to read the chunk from file
* @return dimension data chunk
*/
- @Override public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader,
- int blockIndexes) {
+ @Override public DimensionRawColumnChunk readDimensionChunk(FileReader fileReader,
+ int columnIndex) {
// operation of getting the dimension chunk is not supported as its a
// non leaf node
// and in case of B+Tree data will be stored only in leaf node and
@@ -205,11 +197,11 @@ public class BTreeNonLeafNode implements BTreeNode {
* Below method will be used to get the measure chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndexes block indexes to be read from file
+ * @param columnIndexRange block indexes to be read from file
* @return measure column data chunk
*/
- @Override public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) {
+ @Override public MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) {
// operation of getting the measure chunk is not supported as its a non
// leaf node
// and in case of B+Tree data will be stored only in leaf node and
@@ -222,11 +214,11 @@ public class BTreeNonLeafNode implements BTreeNode {
* Below method will be used to read the measure chunk
*
* @param fileReader file read to read the file chunk
- * @param blockIndex block index to be read from file
+ * @param columnIndex block index to be read from file
* @return measure data chunk
*/
- @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex) {
+ @Override public MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex) {
// operation of getting the measure chunk is not supported as its a non
// leaf node
// and in case of B+Tree data will be stored only in leaf node and
@@ -236,20 +228,6 @@ public class BTreeNonLeafNode implements BTreeNode {
}
/**
- * @return the segmentProperties
- */
- public void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache) {
-
- this.deleteDeltaDataCache = deleteDeltaDataCache;
- }
- /**
- * @return the segmentProperties
- */
- public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
- return deleteDeltaDataCache;
- }
-
- /**
* number of pages in blocklet
* @return
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
index 25817f5..8af7eae 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
@@ -38,7 +38,7 @@ public class BlockBTreeLeafNode extends AbstractBTreeLeafNode {
* node
* @param metadataIndex metadata index
*/
- public BlockBTreeLeafNode(BTreeBuilderInfo builderInfos, int metadataIndex, long nodeNumber) {
+ BlockBTreeLeafNode(BTreeBuilderInfo builderInfos, int metadataIndex, long nodeNumber) {
DataFileFooter footer = builderInfos.getFooterList().get(metadataIndex);
BlockletMinMaxIndex minMaxIndex = footer.getBlockletIndex().getMinMaxIndex();
maxKeyOfColumns = minMaxIndex.getMaxValues();
@@ -63,8 +63,8 @@ public class BlockBTreeLeafNode extends AbstractBTreeLeafNode {
* Below method is suppose to return the Blocklet ID.
* @return
*/
- @Override public String blockletId() {
- return blockInfo.getTableBlockInfo().getDetailInfo().getBlockletId().toString();
+ @Override public short blockletIndex() {
+ return blockInfo.getTableBlockInfo().getDetailInfo().getBlockletId();
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
index 94221ba..ddd7fcf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
@@ -69,7 +69,7 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* this will be used during query execution when we can
* give some leaf node of a btree to one executor some to other
*/
- public BlockletBTreeLeafNode(BTreeBuilderInfo builderInfos, int leafIndex, long nodeNumber) {
+ BlockletBTreeLeafNode(BTreeBuilderInfo builderInfos, int leafIndex, long nodeNumber) {
// get a lead node min max
BlockletMinMaxIndex minMaxIndex =
builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex).getBlockletIndex()
@@ -124,23 +124,23 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
}
}
- @Override public String blockletId() {
- return "0";
+ @Override public short blockletIndex() {
+ return 0;
}
/**
* Below method will be used to get the dimension chunks
*
* @param fileReader file reader to read the chunks from file
- * @param blockIndexes indexes of the blocks need to be read
+ * @param columnIndexRange indexes of the blocks need to be read
* @return dimension data chunks
*/
- @Override public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
if (fileReader.isReadPageByPage()) {
- return dimensionChunksPageLevelReader.readRawDimensionChunks(fileReader, blockIndexes);
+ return dimensionChunksPageLevelReader.readRawDimensionChunks(fileReader, columnIndexRange);
} else {
- return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
+ return dimensionChunksReader.readRawDimensionChunks(fileReader, columnIndexRange);
}
}
@@ -148,15 +148,15 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* Below method will be used to get the dimension chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndex block index to be read
+ * @param columnIndex block index to be read
* @return dimension data chunk
*/
- @Override public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndex)
- throws IOException {
+ @Override public DimensionRawColumnChunk readDimensionChunk(
+ FileReader fileReader, int columnIndex) throws IOException {
if (fileReader.isReadPageByPage()) {
- return dimensionChunksPageLevelReader.readRawDimensionChunk(fileReader, blockIndex);
+ return dimensionChunksPageLevelReader.readRawDimensionChunk(fileReader, columnIndex);
} else {
- return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndex);
+ return dimensionChunksReader.readRawDimensionChunk(fileReader, columnIndex);
}
}
@@ -164,15 +164,15 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* Below method will be used to get the measure chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndexes block indexes to be read from file
+ * @param columnIndexRange block indexes to be read from file
* @return measure column data chunk
*/
- @Override public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
if (fileReader.isReadPageByPage()) {
- return measureColumnChunkPageLevelReader.readRawMeasureChunks(fileReader, blockIndexes);
+ return measureColumnChunkPageLevelReader.readRawMeasureChunks(fileReader, columnIndexRange);
} else {
- return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
+ return measureColumnChunkReader.readRawMeasureChunks(fileReader, columnIndexRange);
}
}
@@ -180,15 +180,15 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* Below method will be used to read the measure chunk
*
* @param fileReader file read to read the file chunk
- * @param blockIndex block index to be read from file
+ * @param columnIndex block index to be read from file
* @return measure data chunk
*/
- @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
+ @Override public MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException {
if (fileReader.isReadPageByPage()) {
- return measureColumnChunkPageLevelReader.readRawMeasureChunk(fileReader, blockIndex);
+ return measureColumnChunkPageLevelReader.readRawMeasureChunk(fileReader, columnIndex);
} else {
- return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
+ return measureColumnChunkReader.readRawMeasureChunk(fileReader, columnIndex);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
index 6d96b3b..597def0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
@@ -41,7 +41,13 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.Encoding;
-import static org.apache.carbondata.format.Encoding.*;
+import static org.apache.carbondata.format.Encoding.ADAPTIVE_DELTA_FLOATING;
+import static org.apache.carbondata.format.Encoding.ADAPTIVE_DELTA_INTEGRAL;
+import static org.apache.carbondata.format.Encoding.ADAPTIVE_FLOATING;
+import static org.apache.carbondata.format.Encoding.ADAPTIVE_INTEGRAL;
+import static org.apache.carbondata.format.Encoding.BOOL_BYTE;
+import static org.apache.carbondata.format.Encoding.DIRECT_COMPRESS;
+import static org.apache.carbondata.format.Encoding.RLE_INTEGRAL;
/**
* Base class for encoding factory implementation.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
index c7411d6..daba470 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
@@ -27,7 +27,6 @@ import org.apache.carbondata.core.dictionary.service.AbstractDictionaryServer;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.util.CarbonProperties;
-
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
index ce05fe2..2865d4b 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
@@ -213,10 +213,6 @@ public class BlockletDetailInfo implements Serializable, Writable {
return columnSchemas;
}
- public void setColumnSchemas(List<ColumnSchema> columnSchemas) {
- this.columnSchemas = columnSchemas;
- }
-
public void setColumnSchemaBinary(byte[] columnSchemaBinary) {
this.columnSchemaBinary = columnSchemaBinary;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
new file mode 100644
index 0000000..4d10fd6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
+import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
+import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+
+/**
+ * wrapper for blocklet data map data
+ */
+public class BlockletDataRefNode implements DataRefNode {
+
+ private List<TableBlockInfo> blockInfos;
+
+ private int index;
+
+ private int[] dimensionLens;
+
+ BlockletDataRefNode(List<TableBlockInfo> blockInfos, int index, int[] dimensionLens) {
+ this.blockInfos = blockInfos;
+ // Update row count and page count to blocklet info
+ for (TableBlockInfo blockInfo : blockInfos) {
+ BlockletDetailInfo detailInfo = blockInfo.getDetailInfo();
+ detailInfo.getBlockletInfo().setNumberOfRows(detailInfo.getRowCount());
+ detailInfo.getBlockletInfo().setNumberOfPages(detailInfo.getPagesCount());
+ detailInfo.setBlockletId(blockInfo.getDetailInfo().getBlockletId());
+ int[] pageRowCount = new int[detailInfo.getPagesCount()];
+ int numberOfPagesCompletelyFilled = detailInfo.getRowCount()
+ / CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ int lastPageRowCount = detailInfo.getRowCount()
+ % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ for (int i = 0; i < numberOfPagesCompletelyFilled; i++) {
+ pageRowCount[i] =
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ }
+ if (lastPageRowCount > 0) {
+ pageRowCount[pageRowCount.length - 1] = lastPageRowCount;
+ }
+ detailInfo.getBlockletInfo().setNumberOfRowsPerPage(pageRowCount);
+ }
+ this.index = index;
+ this.dimensionLens = dimensionLens;
+ }
+
+ @Override public DataRefNode getNextDataRefNode() {
+ if (index + 1 < blockInfos.size()) {
+ return new BlockletDataRefNode(blockInfos, index + 1, dimensionLens);
+ }
+ return null;
+ }
+
+ @Override public int numRows() {
+ return blockInfos.get(index).getDetailInfo().getRowCount();
+ }
+
+ @Override public long nodeIndex() {
+ return index;
+ }
+
+ @Override public short blockletIndex() {
+ return blockInfos.get(index).getDetailInfo().getBlockletId();
+ }
+
+ @Override
+ public byte[][] getColumnsMaxValue() {
+ BlockletIndex blockletIndex =
+ blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
+ // In case of blocklet distribution this will be null
+ if (null != blockletIndex) {
+ return blockletIndex.getMinMaxIndex().getMaxValues();
+ }
+ return null;
+ }
+
+ @Override
+ public byte[][] getColumnsMinValue() {
+ BlockletIndex blockletIndex =
+ blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
+ // In case of blocklet distribution this will be null
+ if (null != blockletIndex) {
+ return blockletIndex.getMinMaxIndex().getMinValues();
+ }
+ return null;
+ }
+
+ @Override
+ public DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader, int[][] blockIndexes)
+ throws IOException {
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
+ return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
+ }
+
+ @Override
+ public DimensionRawColumnChunk readDimensionChunk(FileReader fileReader, int columnIndex)
+ throws IOException {
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
+ return dimensionChunksReader.readRawDimensionChunk(fileReader, columnIndex);
+ }
+
+ @Override
+ public MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader, int[][] columnIndexRange)
+ throws IOException {
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
+ return measureColumnChunkReader.readRawMeasureChunks(fileReader, columnIndexRange);
+ }
+
+ @Override public MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex)
+ throws IOException {
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
+ return measureColumnChunkReader.readRawMeasureChunk(fileReader, columnIndex);
+ }
+
+ private DimensionColumnChunkReader getDimensionColumnChunkReader(FileReader fileReader) {
+ ColumnarFormatVersion version =
+ ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
+ if (fileReader.isReadPageByPage()) {
+ return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
+ blockInfos.get(index).getFilePath(), true);
+ } else {
+ return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
+ blockInfos.get(index).getFilePath(), false);
+ }
+ }
+
+ private MeasureColumnChunkReader getMeasureColumnChunkReader(FileReader fileReader) {
+ ColumnarFormatVersion version =
+ ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
+ if (fileReader.isReadPageByPage()) {
+ return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(),
+ blockInfos.get(index).getFilePath(), true);
+ } else {
+ return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(),
+ blockInfos.get(index).getFilePath(), false);
+ }
+ }
+
+ @Override public int numberOfPages() {
+ return blockInfos.get(index).getDetailInfo().getPagesCount();
+ }
+
+ @Override public int getPageRowCount(int pageNumber) {
+ return blockInfos.get(index).getDetailInfo().getBlockletInfo()
+ .getNumberOfRowsPerPage()[pageNumber];
+ }
+
+ public int numberOfNodes() {
+ return blockInfos.size();
+ }
+
+ public List<TableBlockInfo> getBlockInfos() {
+ return blockInfos;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
deleted file mode 100644
index 097dd8c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore.blockletindex;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
-import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
-import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
-import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
-import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
-
-/**
- * wrapper for blocklet data map data
- */
-public class BlockletDataRefNodeWrapper implements DataRefNode {
-
- private List<TableBlockInfo> blockInfos;
-
- private int index;
-
- private int[] dimensionLens;
-
- private BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache;
-
- public BlockletDataRefNodeWrapper(List<TableBlockInfo> blockInfos, int index,
- int[] dimensionLens) {
- this.blockInfos = blockInfos;
- // Update row count and page count to blocklet info
- for (TableBlockInfo blockInfo : blockInfos) {
- BlockletDetailInfo detailInfo = blockInfo.getDetailInfo();
- detailInfo.getBlockletInfo().setNumberOfRows(detailInfo.getRowCount());
- detailInfo.getBlockletInfo().setNumberOfPages(detailInfo.getPagesCount());
- detailInfo.setBlockletId(blockInfo.getDetailInfo().getBlockletId());
- int[] pageRowCount = new int[detailInfo.getPagesCount()];
- int numberOfPagesCompletelyFilled = detailInfo.getRowCount()
- / CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- int lastPageRowCount = detailInfo.getRowCount()
- % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- for (int i = 0; i < numberOfPagesCompletelyFilled; i++) {
- pageRowCount[i] =
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- }
- if (lastPageRowCount > 0) {
- pageRowCount[pageRowCount.length - 1] = lastPageRowCount;
- }
- detailInfo.getBlockletInfo().setNumberOfRowsPerPage(pageRowCount);
- }
- this.index = index;
- this.dimensionLens = dimensionLens;
- }
-
- @Override public DataRefNode getNextDataRefNode() {
- if (index + 1 < blockInfos.size()) {
- return new BlockletDataRefNodeWrapper(blockInfos, index + 1, dimensionLens);
- }
- return null;
- }
-
- @Override public int nodeSize() {
- return blockInfos.get(index).getDetailInfo().getRowCount();
- }
-
- @Override public long nodeNumber() {
- return index;
- }
-
- @Override public String blockletId() {
- return blockInfos.get(index).getDetailInfo().getBlockletId().toString();
- }
-
- @Override
- public byte[][] getColumnsMaxValue() {
- BlockletIndex blockletIndex =
- blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
- // In case of blocklet distribution this will be null
- if (null != blockletIndex) {
- return blockletIndex.getMinMaxIndex().getMaxValues();
- }
- return null;
- }
-
- @Override
- public byte[][] getColumnsMinValue() {
- BlockletIndex blockletIndex =
- blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
- // In case of blocklet distribution this will be null
- if (null != blockletIndex) {
- return blockletIndex.getMinMaxIndex().getMinValues();
- }
- return null;
- }
-
- @Override
- public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader, int[][] blockIndexes)
- throws IOException {
- DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
- return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
- }
-
- @Override
- public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndexes)
- throws IOException {
- DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
- return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndexes);
- }
-
- @Override
- public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
- throws IOException {
- MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
- return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
- }
-
- @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
- throws IOException {
- MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
- return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
- }
-
- private DimensionColumnChunkReader getDimensionColumnChunkReader(FileHolder fileReader) {
- ColumnarFormatVersion version =
- ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
- if (fileReader.isReadPageByPage()) {
- return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
- blockInfos.get(index).getFilePath(), true);
- } else {
- return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
- blockInfos.get(index).getFilePath(), false);
- }
- }
-
- private MeasureColumnChunkReader getMeasureColumnChunkReader(FileHolder fileReader) {
- ColumnarFormatVersion version =
- ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
- if (fileReader.isReadPageByPage()) {
- return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(),
- blockInfos.get(index).getFilePath(), true);
- } else {
- return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(),
- blockInfos.get(index).getFilePath(), false);
- }
- }
-
- @Override
- public void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache) {
- this.deleteDeltaDataCache = deleteDeltaDataCache;
- }
-
- @Override public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
- return deleteDeltaDataCache;
- }
-
- @Override public int numberOfPages() {
- return blockInfos.get(index).getDetailInfo().getPagesCount();
- }
-
- @Override public int getPageRowCount(int pageNumber) {
- return blockInfos.get(index).getDetailInfo().getBlockletInfo()
- .getNumberOfRowsPerPage()[pageNumber];
- }
-
- public int numberOfNodes() {
- return blockInfos.size();
- }
-
- public List<TableBlockInfo> getBlockInfos() {
- return blockInfos;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
index 17ad17f..a30f64c 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
@@ -32,7 +32,7 @@ public class IndexWrapper extends AbstractIndex {
public IndexWrapper(List<TableBlockInfo> blockInfos) {
segmentProperties = new SegmentProperties(blockInfos.get(0).getDetailInfo().getColumnSchemas(),
blockInfos.get(0).getDetailInfo().getDimLens());
- dataRefNode = new BlockletDataRefNodeWrapper(blockInfos, 0,
+ dataRefNode = new BlockletDataRefNode(blockInfos, 0,
segmentProperties.getDimensionColumnsValueSize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
index a30b04c..a74b5a1 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
@@ -19,7 +19,12 @@ package org.apache.carbondata.core.indexstore.blockletindex;
import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
index 5862933..2203b3b 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
@@ -17,11 +17,11 @@
package org.apache.carbondata.core.memory;
+import javax.annotation.concurrent.GuardedBy;
import java.lang.ref.WeakReference;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
-import javax.annotation.concurrent.GuardedBy;
/**
* Code ported from Apache Spark {org.apache.spark.unsafe.memory} package
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
index 0cb2918..099fffd 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
@@ -29,31 +29,12 @@ public class SegmentInfo implements Serializable {
private static final long serialVersionUID = -1749874611112709431L;
/**
- * number of column in the segment
- */
- private int numberOfColumns;
-
- /**
* cardinality of each columns
* column which is not participating in the multidimensional key cardinality will be -1;
*/
private int[] columnCardinality;
/**
- * @return the numberOfColumns
- */
- public int getNumberOfColumns() {
- return numberOfColumns;
- }
-
- /**
- * @param numberOfColumns the numberOfColumns to set
- */
- public void setNumberOfColumns(int numberOfColumns) {
- this.numberOfColumns = numberOfColumns;
- }
-
- /**
* @return the columnCardinality
*/
public int[] getColumnCardinality() {
[44/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all
path construction through CarbonTablePath static method
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index d96a051..90a4223 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -49,7 +49,7 @@ import org.apache.carbondata.core.scan.partition.PartitionUtil
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager}
import org.apache.carbondata.core.util.{ByteUtil, CarbonProperties, CarbonUtil}
import org.apache.carbondata.core.util.comparator.Comparator
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
@@ -634,7 +634,7 @@ object CommonUtil {
def readLoadMetadataDetails(model: CarbonLoadModel): Unit = {
- val metadataPath = model.getCarbonDataLoadSchema.getCarbonTable.getMetaDataFilepath
+ val metadataPath = model.getCarbonDataLoadSchema.getCarbonTable.getMetadataPath
val details = SegmentStatusManager.readLoadMetadata(metadataPath)
model.setLoadMetadataDetails(new util.ArrayList[LoadMetadataDetails](details.toList.asJava))
}
@@ -848,20 +848,18 @@ object CommonUtil {
val fileType = FileFactory.getFileType(databaseLocation)
if (FileFactory.isFileExist(databaseLocation, fileType)) {
val file = FileFactory.getCarbonFile(databaseLocation, fileType)
- if (file.isDirectory) {
- val tableFolders = file.listFiles()
- tableFolders.foreach { tableFolder =>
- if (tableFolder.isDirectory) {
- val tablePath = databaseLocation +
- CarbonCommonConstants.FILE_SEPARATOR + tableFolder.getName
- val identifier =
- AbsoluteTableIdentifier.from(tablePath, dbName, tableFolder.getName)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier)
- val tableStatusFile = carbonTablePath.getTableStatusFilePath
- if (FileFactory.isFileExist(tableStatusFile, fileType)) {
- val segmentStatusManager = new SegmentStatusManager(identifier)
- val carbonLock = segmentStatusManager.getTableStatusLock
- try {
+ if (file.isDirectory) {
+ val tableFolders = file.listFiles()
+ tableFolders.foreach { tableFolder =>
+ if (tableFolder.isDirectory) {
+ val tablePath = databaseLocation +
+ CarbonCommonConstants.FILE_SEPARATOR + tableFolder.getName
+ val identifier =
+ AbsoluteTableIdentifier.from(tablePath, dbName, tableFolder.getName)
+ val tableStatusFile =
+ CarbonTablePath.getTableStatusFilePath(tablePath)
+ if (FileFactory.isFileExist(tableStatusFile, fileType)) {
+ try {
val carbonTable = CarbonMetadata.getInstance
.getCarbonTable(identifier.getCarbonTableIdentifier.getTableUniqueName)
DataLoadingUtil.deleteLoadsAndUpdateMetadata(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 5e9f7fe..c8331f2 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -344,8 +344,8 @@ object DataLoadingUtil {
def deleteLoadsAndUpdateMetadata(
isForceDeletion: Boolean,
carbonTable: CarbonTable): Unit = {
- if (isLoadDeletionRequired(carbonTable.getMetaDataFilepath)) {
- val details = SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath)
+ if (isLoadDeletionRequired(carbonTable.getMetadataPath)) {
+ val details = SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath)
val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
val carbonTableStatusLock =
CarbonLockFactory.getCarbonLockObj(
@@ -359,7 +359,7 @@ object DataLoadingUtil {
absoluteTableIdentifier,
isForceDeletion,
details,
- carbonTable.getMetaDataFilepath
+ carbonTable.getMetadataPath
)
var updationCompletionStaus = false
@@ -372,7 +372,7 @@ object DataLoadingUtil {
// read latest table status again.
val latestMetadata = SegmentStatusManager
- .readLoadMetadata(carbonTable.getMetaDataFilepath)
+ .readLoadMetadata(carbonTable.getMetadataPath)
// update the metadata details from old to new status.
val latestStatus = CarbonLoaderUtil
@@ -397,7 +397,7 @@ object DataLoadingUtil {
if (updationCompletionStaus) {
DeleteLoadFolders
.physicalFactAndMeasureMetadataDeletion(absoluteTableIdentifier,
- carbonTable.getMetaDataFilepath, isForceDeletion)
+ carbonTable.getMetadataPath, isForceDeletion)
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 5f44e43..bbf345c 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -52,7 +52,7 @@ import org.apache.carbondata.core.reader.CarbonDictionaryReader
import org.apache.carbondata.core.service.CarbonCommonFactory
import org.apache.carbondata.core.statusmanager.SegmentStatus
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, DataTypeUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.writer.CarbonDictionaryWriter
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.csvinput.{CSVInputFormat, StringArrayWritable}
@@ -308,7 +308,7 @@ object GlobalDictionaryUtil {
}
val primDimensions = primDimensionsBuffer.map { x => x }.toArray
val dictDetail = CarbonSparkFactory.getDictionaryDetailService.
- getDictionaryDetail(dictFolderPath, primDimensions, table, carbonLoadModel.getTablePath)
+ getDictionaryDetail(dictFolderPath, primDimensions, carbonLoadModel.getTablePath)
val dictFilePaths = dictDetail.dictFilePaths
val dictFileExists = dictDetail.dictFileExists
val columnIdentifier = dictDetail.columnIdentifiers
@@ -398,10 +398,6 @@ object GlobalDictionaryUtil {
}
}
- // Hack for spark2 integration
- var updateTableMetadataFunc: (CarbonLoadModel, SQLContext, DictionaryLoadModel,
- Array[CarbonDimension]) => Unit = _
-
/**
* check whether global dictionary have been generated successfully or not
*
@@ -705,10 +701,7 @@ object GlobalDictionaryUtil {
try {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val carbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier
- // create dictionary folder if not exists
- val tablePath = carbonLoadModel.getTablePath
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(tablePath, carbonTableIdentifier)
- val dictfolderPath = carbonTablePath.getMetadataDirectoryPath
+ val dictfolderPath = CarbonTablePath.getMetadataPath(carbonLoadModel.getTablePath)
// columns which need to generate global dictionary file
val dimensions = carbonTable.getDimensionByTableName(
carbonTable.getTableName).asScala.toArray
@@ -845,12 +838,11 @@ object GlobalDictionaryUtil {
* This method will write dictionary file, sortindex file and dictionary meta for new dictionary
* column with default value
*
- * @param carbonTablePath
* @param columnSchema
* @param absoluteTableIdentifier
* @param defaultValue
*/
- def loadDefaultDictionaryValueForNewColumn(carbonTablePath: CarbonTablePath,
+ def loadDefaultDictionaryValueForNewColumn(
columnSchema: ColumnSchema,
absoluteTableIdentifier: AbsoluteTableIdentifier,
defaultValue: String): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 9a0098e..3b588df 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.PartitionMapFileStore.PartitionMapper
import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes, DecimalType}
import org.apache.carbondata.core.metadata.encoder.Encoding
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.metadata.schema._
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, RelationIdentifier, TableInfo, TableSchema}
import org.apache.carbondata.core.metadata.schema.table.column.{ColumnSchema, ParentColumnTableRelation}
import org.apache.carbondata.core.service.CarbonCommonFactory
+import org.apache.carbondata.core.service.impl.ColumnUniqueIdGenerator
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentUpdateStatusManager}
import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -207,8 +208,7 @@ class AlterTableColumnSchemaGenerator(
alterTableModel: AlterTableAddColumnsModel,
dbName: String,
tableInfo: TableInfo,
- carbonTablePath: CarbonTablePath,
- tableIdentifier: CarbonTableIdentifier,
+ tableIdentifier: AbsoluteTableIdentifier,
sc: SparkContext) {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
@@ -369,7 +369,7 @@ object TableNewProcessor {
encoders.add(Encoding.DIRECT_DICTIONARY)
}
columnSchema.setEncodingList(encoders)
- val colUniqueIdGenerator = CarbonCommonFactory.getColumnUniqueIdGenerator
+ val colUniqueIdGenerator = ColumnUniqueIdGenerator.getInstance
val columnUniqueId = colUniqueIdGenerator.generateUniqueId(columnSchema)
columnSchema.setColumnUniqueId(columnUniqueId)
columnSchema.setColumnReferenceId(columnUniqueId)
@@ -433,7 +433,7 @@ class TableNewProcessor(cm: TableModel) {
}
}
columnSchema.setEncodingList(encoders)
- val colUniqueIdGenerator = CarbonCommonFactory.getColumnUniqueIdGenerator
+ val colUniqueIdGenerator = ColumnUniqueIdGenerator.getInstance
val columnUniqueId = colUniqueIdGenerator.generateUniqueId(columnSchema)
columnSchema.setColumnUniqueId(columnUniqueId)
columnSchema.setColumnReferenceId(columnUniqueId)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
index 3c871db..1656efa 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
@@ -176,8 +176,6 @@ object PartitionUtils {
getPartitionBlockList(identifier, segmentId, partitionIds, oldPartitionIds,
partitionInfo, carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable).asScala
val pathList: util.List[String] = new util.ArrayList[String]()
- val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
- val carbonTablePath = new CarbonTablePath(carbonTableIdentifier, tablePath)
tableBlockInfoList.foreach{ tableBlockInfo =>
val path = tableBlockInfo.getFilePath
val timestamp = CarbonTablePath.DataFileUtil.getTimeStampFromFileName(path)
@@ -190,8 +188,8 @@ object PartitionUtils {
val batchNo = CarbonTablePath.DataFileUtil.getBatchNoFromTaskNo(taskNo)
val taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskNo)
val bucketNumber = CarbonTablePath.DataFileUtil.getBucketNo(path)
- val indexFilePath = carbonTablePath.getCarbonIndexFilePath(
- String.valueOf(taskId), segmentId, batchNo, String.valueOf(bucketNumber),
+ val indexFilePath = CarbonTablePath.getCarbonIndexFilePath(
+ tablePath, String.valueOf(taskId), segmentId, batchNo, String.valueOf(bucketNumber),
timestamp, version)
// indexFilePath could be duplicated when multiple data file related to one index file
if (indexFilePath != null && !pathList.contains(indexFilePath)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
index 5f8f389..adf33ff 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.OperationContext
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CompactionType}
@@ -70,7 +70,7 @@ class AggregateDataMapCompactor(carbonLoadModel: CarbonLoadModel,
sqlContext.sparkSession, loadCommand.logicalPlan.get))
loadCommand.processData(sqlContext.sparkSession)
val newLoadMetaDataDetails = SegmentStatusManager.readLoadMetadata(
- carbonTable.getMetaDataFilepath)
+ carbonTable.getMetadataPath)
val updatedLoadMetaDataDetails = newLoadMetaDataDetails collect {
case load if loadMetaDataDetails.contains(load) =>
load.setMergedLoadName(mergedLoadName)
@@ -79,12 +79,9 @@ class AggregateDataMapCompactor(carbonLoadModel: CarbonLoadModel,
load
case other => other
}
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- .getAbsoluteTableIdentifier)
- SegmentStatusManager
- .writeLoadDetailsIntoFile(carbonTablePath.getTableStatusFilePath,
- updatedLoadMetaDataDetails)
+ SegmentStatusManager.writeLoadDetailsIntoFile(
+ CarbonTablePath.getTableStatusFilePath(carbonLoadModel.getTablePath),
+ updatedLoadMetaDataDetails)
carbonLoadModel.setLoadMetadataDetails(updatedLoadMetaDataDetails.toList.asJava)
} finally {
// check if any other segments needs compaction on in case of MINOR_COMPACTION.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 655e5a0..f47c9bc 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -57,7 +57,7 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.scan.partition.PartitionUtil
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.{ByteUtil, CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.FailureCauses
@@ -70,7 +70,7 @@ import org.apache.carbondata.processing.merger.{CarbonCompactionUtil, CarbonData
import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil, CarbonLoaderUtil}
import org.apache.carbondata.spark.{DataLoadResultImpl, PartitionFactory, _}
import org.apache.carbondata.spark.load._
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, DataLoadingUtil, Util}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, Util}
/**
* This is the factory class which can create different RDD depends on user needs.
@@ -137,7 +137,7 @@ object CarbonDataRDDFactory {
LOGGER.error("Not able to acquire the compaction lock for table " +
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
CarbonCompactionUtil
- .createCompactionRequiredFile(carbonTable.getMetaDataFilepath, compactionType)
+ .createCompactionRequiredFile(carbonTable.getMetadataPath, compactionType)
// throw exception only in case of DDL trigger.
if (compactionModel.isDDLTrigger) {
CarbonException.analysisException(
@@ -205,7 +205,7 @@ object CarbonDataRDDFactory {
s"${ tableForCompaction.getDatabaseName }." +
s"${ tableForCompaction.getTableName}")
val table: CarbonTable = tableForCompaction
- val metadataPath = table.getMetaDataFilepath
+ val metadataPath = table.getMetadataPath
val compactionType = CarbonCompactionUtil.determineCompactionType(metadataPath)
val newCarbonLoadModel = prepareCarbonLoadModel(table)
@@ -596,15 +596,13 @@ object CarbonDataRDDFactory {
(row.get(row.size - 1).toString, Row(row.toSeq.slice(0, row.size - 1): _*)))
val loadMetadataDetails = SegmentStatusManager.readLoadMetadata(
- carbonTable.getMetaDataFilepath)
+ carbonTable.getMetadataPath)
.filter(lmd => lmd.getSegmentStatus.equals(SegmentStatus.LOAD_PARTIAL_SUCCESS) ||
lmd.getSegmentStatus.equals(SegmentStatus.SUCCESS))
val segmentIds = loadMetadataDetails.map(_.getLoadName)
val segmentIdIndex = segmentIds.zipWithIndex.toMap
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonLoadModel.getTablePath,
- carbonTable.getCarbonTableIdentifier)
val segmentId2maxTaskNo = segmentIds.map { segId =>
- (segId, CarbonUpdateUtil.getLatestTaskIdForSegment(segId, carbonTablePath))
+ (segId, CarbonUpdateUtil.getLatestTaskIdForSegment(segId, carbonLoadModel.getTablePath))
}.toMap
class SegmentPartitioner(segIdIndex: Map[String, Int], parallelism: Int)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index a0c8f65..ddc8586 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -156,7 +156,7 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
}
val carbonMergerMapping = CarbonMergerMapping(
tablePath,
- carbonTable.getMetaDataFilepath,
+ carbonTable.getMetadataPath,
mergedLoadName,
databaseName,
factTableName,
@@ -169,7 +169,7 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
partitionMapper)
carbonLoadModel.setTablePath(carbonMergerMapping.hdfsStoreLocation)
carbonLoadModel.setLoadMetadataDetails(
- SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath).toList.asJava)
+ SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath).toList.asJava)
// trigger event for compaction
val alterTableCompactionPreEvent: AlterTableCompactionPreEvent =
AlterTableCompactionPreEvent(sqlContext.sparkSession,
@@ -240,10 +240,10 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
((compactionType == CompactionType.IUD_UPDDEL_DELTA) &&
CarbonDataMergerUtil
.updateLoadMetadataIUDUpdateDeltaMergeStatus(loadsToMerge,
- carbonTable.getMetaDataFilepath,
+ carbonTable.getMetadataPath,
carbonLoadModel)) ||
CarbonDataMergerUtil
- .updateLoadMetadataWithMergeStatus(loadsToMerge, carbonTable.getMetaDataFilepath,
+ .updateLoadMetadataWithMergeStatus(loadsToMerge, carbonTable.getMetadataPath,
mergedLoadNumber, carbonLoadModel, compactionType)
if (!statusFileUpdation) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index fb0f9fe..febb83e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -37,8 +37,9 @@ import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.CarbonUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{AlterTableCompactionPostEvent, AlterTableCompactionPreEvent, AlterTableCompactionPreStatusUpdateEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.processing.loading.events.LoadEvents.LoadMetadataEvent
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
@@ -237,7 +238,7 @@ case class CarbonAlterTableCompactionCommand(
readFileFooterFromCarbonDataFile = true)
val carbonMergerMapping = CarbonMergerMapping(carbonTable.getTablePath,
- carbonTable.getMetaDataFilepath,
+ carbonTable.getMetadataPath,
"",
carbonTable.getDatabaseName,
carbonTable.getTableName,
@@ -312,9 +313,10 @@ case class CarbonAlterTableCompactionCommand(
true)(sparkSession,
sparkSession.sessionState.catalog.asInstanceOf[CarbonSessionCatalog])
// 5. remove checkpoint
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- FileFactory.deleteAllFilesOfDir(new File(tablePath.getStreamingCheckpointDir))
- FileFactory.deleteAllFilesOfDir(new File(tablePath.getStreamingLogDir))
+ FileFactory.deleteAllFilesOfDir(
+ new File(CarbonTablePath.getStreamingCheckpointDir(carbonTable.getTablePath)))
+ FileFactory.deleteAllFilesOfDir(
+ new File(CarbonTablePath.getStreamingLogDir(carbonTable.getTablePath)))
} else {
val msg = "Failed to close streaming table, because streaming is locked for table " +
carbonTable.getDatabaseName() + "." + carbonTable.getTableName()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 226a625..c7b59d4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -59,7 +59,7 @@ import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.events.exception.PreEventException
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil
@@ -67,8 +67,8 @@ import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.csvinput.{CSVInputFormat, StringArrayWritable}
import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadMetadataEvent, LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
-import org.apache.carbondata.processing.loading.exception.{NoRetryException}
-import org.apache.carbondata.processing.loading.model.{CarbonLoadModel}
+import org.apache.carbondata.processing.loading.exception.NoRetryException
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.util.CarbonLoaderUtil
import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
@@ -217,8 +217,7 @@ case class CarbonLoadDataCommand(
carbonLoadModel.setUseOnePass(false)
}
// Create table and metadata folders if not exist
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier)
- val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
+ val metadataDirectoryPath = CarbonTablePath.getMetadataPath(table.getTablePath)
val fileType = FileFactory.getFileType(metadataDirectoryPath)
if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
FileFactory.mkdirs(metadataDirectoryPath, fileType)
@@ -309,9 +308,7 @@ case class CarbonLoadDataCommand(
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val carbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier
.getCarbonTableIdentifier
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(carbonLoadModel.getTablePath, carbonTableIdentifier)
- val dictFolderPath = carbonTablePath.getMetadataDirectoryPath
+ val dictFolderPath = CarbonTablePath.getMetadataPath(carbonLoadModel.getTablePath)
val dimensions = carbonTable.getDimensionByTableName(
carbonTable.getTableName).asScala.toArray
val colDictFilePath = carbonLoadModel.getColDictFilePath
@@ -880,4 +877,5 @@ case class CarbonLoadDataCommand(
val dataFrameWithTupleId = dataFrame.get.select(fieldWithTupleId: _*)
(dataFrameWithTupleId)
}
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala
index f8f215f..1e5885e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonShowLoadsCommand.scala
@@ -45,7 +45,7 @@ case class CarbonShowLoadsCommand(
val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
CarbonStore.showSegments(
limit,
- carbonTable.getMetaDataFilepath
+ carbonTable.getMetadataPath
)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
index 2983ea4..90ff3b4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
@@ -35,7 +35,7 @@ import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.{DataMapSchema, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus, RefreshTablePostExecutionEvent, RefreshTablePreExecutionEvent}
import org.apache.carbondata.hadoop.util.SchemaReader
@@ -62,19 +62,18 @@ case class RefreshCarbonTableCommand(
// 2.2.1 validate that all the aggregate tables are copied at the store location.
// 2.2.2 Register the aggregate tables
val tablePath = CarbonEnv.getTablePath(databaseNameOp, tableName)(sparkSession)
- val absoluteTableIdentifier = AbsoluteTableIdentifier.from(tablePath, databaseName, tableName)
+ val identifier = AbsoluteTableIdentifier.from(tablePath, databaseName, tableName)
// 2.1 check if the table already register with hive then ignore and continue with the next
// schema
if (!sparkSession.sessionState.catalog.listTables(databaseName)
.exists(_.table.equalsIgnoreCase(tableName))) {
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)
// check the existence of the schema file to know its a carbon table
- val schemaFilePath = carbonTablePath.getSchemaFilePath
+ val schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath)
// if schema file does not exist then the table will either non carbon table or stale
// carbon table
if (FileFactory.isFileExist(schemaFilePath, FileFactory.getFileType(schemaFilePath))) {
// read TableInfo
- val tableInfo = SchemaReader.getTableInfo(absoluteTableIdentifier)
+ val tableInfo = SchemaReader.getTableInfo(identifier)
// 2.2 register the table with the hive check if the table being registered has
// aggregate table then do the below steps
// 2.2.1 validate that all the aggregate tables are copied at the store location.
@@ -98,7 +97,7 @@ case class RefreshCarbonTableCommand(
// Register partitions to hive metastore in case of hive partitioning carbon table
if (tableInfo.getFactTable.getPartitionInfo != null &&
tableInfo.getFactTable.getPartitionInfo.getPartitionType == PartitionType.NATIVE_HIVE) {
- registerAllPartitionsToHive(absoluteTableIdentifier, sparkSession)
+ registerAllPartitionsToHive(identifier, sparkSession)
}
} else {
LOGGER.audit(
@@ -177,9 +176,7 @@ case class RefreshCarbonTableCommand(
dataMapSchemaList.asScala.foreach(dataMap => {
val tableName = dataMap.getChildSchema.getTableName
val tablePath = CarbonEnv.getTablePath(Some(dbName), tableName)(sparkSession)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(tablePath,
- new CarbonTableIdentifier(dbName, tableName, dataMap.getChildSchema.getTableId))
- val schemaFilePath = carbonTablePath.getSchemaFilePath
+ val schemaFilePath = CarbonTablePath.getSchemaFilePath(tablePath)
try {
fileExist = FileFactory.isFileExist(schemaFilePath, FileFactory.getFileType(schemaFilePath))
} catch {
@@ -190,7 +187,7 @@ case class RefreshCarbonTableCommand(
return fileExist;
}
})
- return true
+ true
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
index a8efb84..58456b7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
@@ -40,7 +40,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, DeleteDeltaBlockDetails, SegmentUpdateDetails, TupleIdEnum}
import org.apache.carbondata.core.mutate.data.RowCountDetailsVO
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentUpdateStatusManager}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
import org.apache.carbondata.processing.exception.MultipleMatchingException
@@ -67,8 +67,7 @@ object DeleteExecution {
val database = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)
- val factPath = carbonTablePath.getFactDir
+ val factPath = CarbonTablePath.getFactDir(carbonTable.getTablePath)
var segmentsTobeDeleted = Seq.empty[String]
val deleteRdd = if (isUpdateOperation) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
index 114c25d..b53c609 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
@@ -39,7 +39,7 @@ import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.spark.partition.DropPartitionCallable
@@ -69,8 +69,8 @@ case class CarbonAlterTableDropPartitionCommand(
LOGGER.error(s"Alter table failed. table not found: $dbName.$tableName")
sys.error(s"Alter table failed. table not found: $dbName.$tableName")
}
- val table = relation.carbonTable
- val partitionInfo = table.getPartitionInfo(tableName)
+ val carbonTable = relation.carbonTable
+ val partitionInfo = carbonTable.getPartitionInfo(tableName)
if (partitionInfo == null) {
sys.error(s"Table $tableName is not a partition table.")
}
@@ -95,11 +95,9 @@ case class CarbonAlterTableDropPartitionCommand(
sys.error(s"Dropping range interval partition isn't support yet!")
}
partitionInfo.dropPartition(partitionIndex)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier)
- val schemaFilePath = carbonTablePath.getSchemaFilePath
- // read TableInfo
- val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ // read TableInfo
+ val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTable)(sparkSession)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo,
dbName, tableName, tablePath)
@@ -112,11 +110,11 @@ case class CarbonAlterTableDropPartitionCommand(
thriftTable.getFact_table.getSchema_evolution.getSchema_evolution_history.get(0)
.setTime_stamp(System.currentTimeMillis)
carbonMetaStore.updateTableSchemaForAlter(
- table.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- table.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
+ carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
+ carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
thriftTable,
null,
- table.getAbsoluteTableIdentifier.getTablePath)(sparkSession)
+ carbonTable.getAbsoluteTableIdentifier.getTablePath)(sparkSession)
// update the schema modified time
carbonMetaStore.updateAndTouchSchemasUpdatedTime()
// sparkSession.catalog.refreshTable(tableName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
index bafc96a..84779cc 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
@@ -41,7 +41,7 @@ import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.spark.partition.SplitPartitionCallable
@@ -72,8 +72,8 @@ case class CarbonAlterTableSplitPartitionCommand(
LOGGER.error(s"Alter table failed. table not found: $dbName.$tableName")
sys.error(s"Alter table failed. table not found: $dbName.$tableName")
}
- val table = relation.carbonTable
- val partitionInfo = table.getPartitionInfo(tableName)
+ val carbonTable = relation.carbonTable
+ val partitionInfo = carbonTable.getPartitionInfo(tableName)
val partitionIds = partitionInfo.getPartitionIds.asScala.map(_.asInstanceOf[Int]).toList
// keep a copy of partitionIdList before update partitionInfo.
// will be used in partition data scan
@@ -88,10 +88,8 @@ case class CarbonAlterTableSplitPartitionCommand(
updatePartitionInfo(partitionInfo, partitionIds)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(table.getAbsoluteTableIdentifier)
- val schemaFilePath = carbonTablePath.getSchemaFilePath
// read TableInfo
- val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTable)(sparkSession)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo,
dbName, tableName, tablePath)
@@ -101,12 +99,12 @@ case class CarbonAlterTableSplitPartitionCommand(
wrapperTableInfo.setLastUpdatedTime(System.currentTimeMillis())
val thriftTable =
schemaConverter.fromWrapperToExternalTableInfo(wrapperTableInfo, dbName, tableName)
- carbonMetaStore
- .updateTableSchemaForAlter(table.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- table.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- thriftTable,
- null,
- table.getAbsoluteTableIdentifier.getTablePath)(sparkSession)
+ carbonMetaStore.updateTableSchemaForAlter(
+ carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
+ carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
+ thriftTable,
+ null,
+ carbonTable.getAbsoluteTableIdentifier.getTablePath)(sparkSession)
// update the schema modified time
carbonMetaStore.updateAndTouchSchemasUpdatedTime()
Seq.empty
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index dbbf90c..fed4235 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -183,7 +183,7 @@ case class CreatePreAggregateTableCommand(
// need to fire load for pre-aggregate table. Therefore reading the load details for PARENT
// table.
DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false, parentTable)
- val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetaDataFilepath)
+ val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetadataPath)
if (loadAvailable.exists(load => load.getSegmentStatus == SegmentStatus.INSERT_IN_PROGRESS ||
load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) {
throw new UnsupportedOperationException(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index dac5d5e..feef7a1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -16,11 +16,12 @@
*/
package org.apache.spark.sql.execution.command.preaaggregate
-import scala.collection.mutable.{ArrayBuffer, ListBuffer}
import scala.collection.JavaConverters._
+import scala.collection.mutable.{ArrayBuffer, ListBuffer}
-import org.apache.spark.sql._
-import org.apache.spark.sql.CarbonExpressions.{CarbonSubqueryAlias => SubqueryAlias, MatchCastExpression}
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, CarbonSession, SparkSession, _}
+import org.apache.spark.sql.CarbonExpressions.{CarbonSubqueryAlias => SubqueryAlias}
+import org.apache.spark.sql.CarbonExpressions.MatchCastExpression
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedFunction, UnresolvedRelation}
import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, AttributeSeq, Cast, Expression, ExprId, NamedExpression, ScalaUDF}
@@ -40,7 +41,6 @@ import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverte
import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable, DataMapSchema, TableSchema}
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.util.CarbonUtil
-import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.format.TableInfo
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -423,9 +423,7 @@ object PreAggregateUtil {
locks = acquireLock(dbName, tableName, locksToBeAcquired, carbonTable)
// get the latest carbon table and check for column existence
// read the latest schema file
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(
- carbonTable.getAbsoluteTableIdentifier)
- val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
thriftTableInfo,
@@ -528,8 +526,7 @@ object PreAggregateUtil {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
carbonTable.getTableLastUpdatedTime
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
if (thriftTable.dataMapSchemas.size > numberOfChildSchema) {
metastore.revertTableSchemaForPreAggCreationFailure(
carbonTable.getAbsoluteTableIdentifier, thriftTable)(sparkSession)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
index f3f01bb..07917d0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
@@ -28,7 +28,6 @@ import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.events.{AlterTableAddColumnPostEvent, AlterTableAddColumnPreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.format.TableInfo
import org.apache.carbondata.spark.rdd.{AlterTableAddColumnRDD, AlterTableDropColumnRDD}
@@ -64,9 +63,7 @@ private[sql] case class CarbonAlterTableAddColumnCommand(
OperationListenerBus.getInstance().fireEvent(alterTableAddColumnListener, operationContext)
// get the latest carbon table and check for column existence
// read the latest schema file
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter
.fromExternalToWrapperTableInfo(thriftTableInfo,
@@ -76,8 +73,7 @@ private[sql] case class CarbonAlterTableAddColumnCommand(
newCols = new AlterTableColumnSchemaGenerator(alterTableAddColumnsModel,
dbName,
wrapperTableInfo,
- carbonTablePath,
- carbonTable.getCarbonTableIdentifier,
+ carbonTable.getAbsoluteTableIdentifier,
sparkSession.sparkContext).process
// generate dictionary files for the newly added columns
new AlterTableAddColumnRDD(sparkSession.sparkContext,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
index 9bea935..fa8003e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
@@ -27,7 +27,7 @@ import org.apache.spark.util.AlterTableUtil
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{AlterTableDataTypeChangePostEvent, AlterTableDataTypeChangePreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.format.{ColumnSchema, SchemaEvolutionEntry, TableInfo}
import org.apache.carbondata.spark.util.{CarbonScalaUtil, DataTypeConverterUtil}
@@ -74,9 +74,7 @@ private[sql] case class CarbonAlterTableDataTypeChangeCommand(
sys.error(s"Invalid Column: $columnName")
}
// read the latest schema file
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val tableInfo: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ val tableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
// maintain the added column for schema evolution history
var addColumnSchema: ColumnSchema = null
var deletedColumnSchema: ColumnSchema = null
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
index 0319d9e..d848eb5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
@@ -29,7 +29,7 @@ import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{AlterTableDropColumnPostEvent, AlterTableDropColumnPreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.format.SchemaEvolutionEntry
import org.apache.carbondata.spark.rdd.AlterTableDropColumnRDD
@@ -98,10 +98,8 @@ private[sql] case class CarbonAlterTableDropColumnCommand(
OperationListenerBus.getInstance().fireEvent(alterTableDropColumnPreEvent, operationContext)
// read the latest schema file
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
val tableInfo: org.apache.carbondata.format.TableInfo =
- metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ metastore.getThriftTableInfo(carbonTable)(sparkSession)
// maintain the deleted columns for schema evolution history
var deletedColumnSchema = ListBuffer[org.apache.carbondata.format.ColumnSchema]()
val columnSchemaList = tableInfo.fact_table.table_columns.asScala
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index dd34f08..7a56dbf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -34,7 +34,7 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonUtil
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{AlterTableRenamePostEvent, AlterTableRenamePreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.format.SchemaEvolutionEntry
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
@@ -98,8 +98,7 @@ private[sql] case class CarbonAlterTableRenameCommand(
val oldTableIdentifier = carbonTable.getAbsoluteTableIdentifier
DataMapStoreManager.getInstance().clearDataMaps(oldTableIdentifier)
// get the latest carbon table and check for column existence
- val oldTablePath = CarbonStorePath.getCarbonTablePath(oldTableIdentifier)
- val tableMetadataFile = oldTablePath.getPath
+ val tableMetadataFile = oldTableIdentifier.getTablePath
val operationContext = new OperationContext
// TODO: Pass new Table Path in pre-event.
val alterTableRenamePreEvent: AlterTableRenamePreEvent = AlterTableRenamePreEvent(
@@ -109,7 +108,7 @@ private[sql] case class CarbonAlterTableRenameCommand(
sparkSession)
OperationListenerBus.getInstance().fireEvent(alterTableRenamePreEvent, operationContext)
val tableInfo: org.apache.carbondata.format.TableInfo =
- metastore.getThriftTableInfo(oldTablePath)(sparkSession)
+ metastore.getThriftTableInfo(carbonTable)(sparkSession)
val schemaEvolutionEntry = new SchemaEvolutionEntry(System.currentTimeMillis)
schemaEvolutionEntry.setTableName(newTableName)
timeStamp = System.currentTimeMillis()
@@ -118,7 +117,8 @@ private[sql] case class CarbonAlterTableRenameCommand(
val fileType = FileFactory.getFileType(tableMetadataFile)
val newTableIdentifier = new CarbonTableIdentifier(oldDatabaseName,
newTableName, carbonTable.getCarbonTableIdentifier.getTableId)
- var newTablePath = CarbonUtil.getNewTablePath(oldTablePath, newTableIdentifier.getTableName)
+ var newTablePath = CarbonTablePath.getNewTablePath(
+ oldTableIdentifier.getTablePath, newTableIdentifier.getTableName)
metastore.removeTableFromMetadata(oldDatabaseName, oldTableName)
val hiveClient = sparkSession.sessionState.catalog.asInstanceOf[CarbonSessionCatalog]
.getClient()
@@ -133,9 +133,9 @@ private[sql] case class CarbonAlterTableRenameCommand(
// changed the rename order to deal with situation when carbon table and hive table
// will point to the same tablePath
if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
- val rename = FileFactory.getCarbonFile(oldTablePath.getPath, fileType)
- .renameForce(oldTablePath.getParent.toString + CarbonCommonConstants.FILE_SEPARATOR +
- newTableName)
+ val rename = FileFactory.getCarbonFile(oldTableIdentifier.getTablePath, fileType)
+ .renameForce(
+ CarbonTablePath.getNewTablePath(oldTableIdentifier.getTablePath, newTableName))
if (!rename) {
renameBadRecords(newTableName, oldTableName, oldDatabaseName)
sys.error(s"Folder rename failed for table $oldDatabaseName.$oldTableName")
@@ -163,13 +163,11 @@ private[sql] case class CarbonAlterTableRenameCommand(
case e: Exception =>
LOGGER.error(e, "Rename table failed: " + e.getMessage)
if (carbonTable != null) {
- AlterTableUtil
- .revertRenameTableChanges(oldTableIdentifier,
- newTableName,
- carbonTable.getTablePath,
- carbonTable.getCarbonTableIdentifier.getTableId,
- timeStamp)(
- sparkSession)
+ AlterTableUtil.revertRenameTableChanges(
+ newTableName,
+ carbonTable,
+ timeStamp)(
+ sparkSession)
renameBadRecords(newTableName, oldTableName, oldDatabaseName)
}
sys.error(s"Alter table rename table operation failed: ${e.getMessage}")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index b44dc7e..fd09e48 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -46,7 +46,8 @@ import org.apache.carbondata.core.metadata.schema
import org.apache.carbondata.core.metadata.schema.table
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.core.util.path.CarbonTablePath.getNewTablePath
import org.apache.carbondata.core.writer.ThriftWriter
import org.apache.carbondata.events.{LookupRelationPostEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
@@ -209,11 +210,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
val dbName = identifier.getCarbonTableIdentifier.getDatabaseName
val tableName = identifier.getCarbonTableIdentifier.getTableName
val tablePath = identifier.getTablePath
- val carbonTableIdentifier = new CarbonTableIdentifier(dbName.toLowerCase(),
- tableName.toLowerCase(), UUID.randomUUID().toString)
- val carbonTablePath =
- CarbonStorePath.getCarbonTablePath(tablePath, carbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
+ val tableMetadataFile = CarbonTablePath.getSchemaFilePath(tablePath)
val fileType = FileFactory.getFileType(tableMetadataFile)
if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
val tableUniqueName = CarbonTable.buildUniqueName(dbName, tableName)
@@ -240,13 +237,13 @@ class CarbonFileMetastore extends CarbonMetaStore {
thriftTableInfo: org.apache.carbondata.format.TableInfo,
schemaEvolutionEntry: SchemaEvolutionEntry,
tablePath: String) (sparkSession: SparkSession): String = {
- val absoluteTableIdentifier = AbsoluteTableIdentifier.from(tablePath, oldTableIdentifier)
+ val identifier = AbsoluteTableIdentifier.from(tablePath, oldTableIdentifier)
val schemaConverter = new ThriftWrapperSchemaConverterImpl
if (schemaEvolutionEntry != null) {
thriftTableInfo.fact_table.schema_evolution.schema_evolution_history.add(schemaEvolutionEntry)
}
- val oldTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)
- val newTablePath = CarbonUtil.getNewTablePath(oldTablePath, newTableIdentifier.getTableName)
+ val newTablePath = CarbonTablePath.getNewTablePath(
+ identifier.getTablePath, newTableIdentifier.getTableName)
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
thriftTableInfo,
newTableIdentifier.getDatabaseName,
@@ -341,8 +338,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
private def createSchemaThriftFile(
identifier: AbsoluteTableIdentifier,
thriftTableInfo: TableInfo): String = {
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier)
- val schemaFilePath = carbonTablePath.getSchemaFilePath
+ val schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath)
val schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath)
val fileType = FileFactory.getFileType(schemaMetadataPath)
if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
@@ -356,7 +352,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
thriftWriter.write(thriftTableInfo)
thriftWriter.close()
updateSchemasUpdatedTime(touchSchemaFileSystemTime())
- carbonTablePath.getPath
+ identifier.getTablePath
}
protected def addTableCache(
@@ -431,8 +427,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
(sparkSession: SparkSession) {
val dbName = absoluteTableIdentifier.getCarbonTableIdentifier.getDatabaseName
val tableName = absoluteTableIdentifier.getCarbonTableIdentifier.getTableName
- val metadataFilePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)
- .getMetadataDirectoryPath
+ val metadataFilePath = CarbonTablePath.getMetadataPath(absoluteTableIdentifier.getTablePath)
val carbonTable = CarbonMetadata.getInstance.getCarbonTable(dbName, tableName)
if (null != carbonTable) {
// clear driver B-tree and dictionary cache
@@ -528,9 +523,9 @@ class CarbonFileMetastore extends CarbonMetaStore {
override def listAllTables(sparkSession: SparkSession): Seq[CarbonTable] =
metadata.carbonTables
- override def getThriftTableInfo(tablePath: CarbonTablePath)
+ override def getThriftTableInfo(carbonTable: CarbonTable)
(sparkSession: SparkSession): TableInfo = {
- val tableMetadataFile = tablePath.getSchemaFilePath
+ val tableMetadataFile = CarbonTablePath.getSchemaFilePath(carbonTable.getTablePath)
CarbonUtil.readSchemaFile(tableMetadataFile)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
index 759471b..44f731e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
@@ -28,7 +28,7 @@ import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetad
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonUtil
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.format
import org.apache.carbondata.format.SchemaEvolutionEntry
import org.apache.carbondata.spark.util.CarbonSparkUtil
@@ -96,12 +96,8 @@ class CarbonHiveMetaStore extends CarbonFileMetastore {
Seq()
}
- override def getThriftTableInfo(tablePath: CarbonTablePath)
+ override def getThriftTableInfo(carbonTable: CarbonTable)
(sparkSession: SparkSession): format.TableInfo = {
- val identifier = tablePath.getCarbonTableIdentifier
- val relation = lookupRelation(TableIdentifier(identifier.getTableName,
- Some(identifier.getDatabaseName)))(sparkSession).asInstanceOf[CarbonRelation]
- val carbonTable = relation.metaData.carbonTable
val schemaConverter = new ThriftWrapperSchemaConverterImpl
schemaConverter.fromWrapperToExternalTableInfo(carbonTable.getTableInfo,
carbonTable.getDatabaseName,
@@ -148,7 +144,8 @@ class CarbonHiveMetaStore extends CarbonFileMetastore {
thriftTableInfo: org.apache.carbondata.format.TableInfo,
carbonTablePath: String)(sparkSession: SparkSession): String = {
val schemaConverter = new ThriftWrapperSchemaConverterImpl
- updateHiveMetaStoreForAlter(newTableIdentifier,
+ updateHiveMetaStoreForAlter(
+ newTableIdentifier,
oldTableIdentifier,
thriftTableInfo,
carbonTablePath,
@@ -163,7 +160,7 @@ class CarbonHiveMetaStore extends CarbonFileMetastore {
sparkSession: SparkSession,
schemaConverter: ThriftWrapperSchemaConverterImpl) = {
val newTablePath =
- CarbonUtil.getNewTablePath(new Path(oldTablePath), newTableIdentifier.getTableName)
+ CarbonTablePath.getNewTablePath(oldTablePath, newTableIdentifier.getTableName)
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
thriftTableInfo,
newTableIdentifier.getDatabaseName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
index 93c7c09..0645040 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -143,7 +143,7 @@ trait CarbonMetaStore {
def listAllTables(sparkSession: SparkSession): Seq[CarbonTable]
- def getThriftTableInfo(tablePath: CarbonTablePath)(sparkSession: SparkSession): TableInfo
+ def getThriftTableInfo(carbonTable: CarbonTable)(sparkSession: SparkSession): TableInfo
def getTableFromMetadataCache(database: String, tableName: String): Option[CarbonTable]
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
index b8608f4..a722838 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
@@ -34,7 +34,7 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
/**
* Represents logical plan for one carbon table
@@ -209,9 +209,7 @@ case class CarbonRelation(
.getValidAndInvalidSegments.getValidSegments.isEmpty) {
sizeInBytesLocalValue = 0L
} else {
- val tablePath = CarbonStorePath.getCarbonTablePath(
- carbonTable.getTablePath,
- carbonTable.getCarbonTableIdentifier).getPath
+ val tablePath = carbonTable.getTablePath
val fileType = FileFactory.getFileType(tablePath)
if (FileFactory.isFileExist(tablePath, fileType)) {
// get the valid segments
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index 8ebd5a9..bc36e9c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -36,7 +36,8 @@ import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTable
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonUtil
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.core.util.path.CarbonTablePath.getNewTablePath
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
@@ -176,41 +177,28 @@ object AlterTableUtil {
/**
* This method reverts the changes to the schema if the rename table command fails.
- *
- * @param oldTableIdentifier
- * @param newTableName
- * @param timeStamp
- * @param sparkSession
*/
- def revertRenameTableChanges(oldTableIdentifier: TableIdentifier,
+ def revertRenameTableChanges(
newTableName: String,
- tablePath: String,
- tableId: String,
+ oldCarbonTable: CarbonTable,
timeStamp: Long)
(sparkSession: SparkSession): Unit = {
- val database = oldTableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase)
- val oldCarbonTableIdentifier = new CarbonTableIdentifier(database,
- oldTableIdentifier.table, tableId)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(tablePath, oldCarbonTableIdentifier)
+ val tablePath = oldCarbonTable.getTablePath
+ val tableId = oldCarbonTable.getCarbonTableIdentifier.getTableId
+ val oldCarbonTableIdentifier = oldCarbonTable.getCarbonTableIdentifier
+ val database = oldCarbonTable.getDatabaseName
val newCarbonTableIdentifier = new CarbonTableIdentifier(database, newTableName, tableId)
- val newTablePath = CarbonUtil.getNewTablePath(new Path(tablePath), newTableName)
+ val newTablePath = CarbonTablePath.getNewTablePath(tablePath, newTableName)
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val fileType = FileFactory.getFileType(tablePath)
if (FileFactory.isFileExist(tablePath, fileType)) {
- val tableInfo = if (metastore.isReadFromHiveMetaStore) {
- // In case of hive metastore we first update the carbonschema inside old table only.
- metastore.getThriftTableInfo(CarbonStorePath.getCarbonTablePath(tablePath,
- new CarbonTableIdentifier(database, oldTableIdentifier.table, tableId)))(sparkSession)
- } else {
- metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
- }
+ val tableInfo = metastore.getThriftTableInfo(oldCarbonTable)(sparkSession)
val evolutionEntryList = tableInfo.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
- LOGGER.error(s"Reverting changes for $database.${ oldTableIdentifier.table }")
- FileFactory.getCarbonFile(carbonTablePath.getPath, fileType)
- .renameForce(carbonTablePath.getParent.toString + CarbonCommonConstants.FILE_SEPARATOR +
- oldTableIdentifier.table)
+ LOGGER.error(s"Reverting changes for $database.${oldCarbonTable.getTableName}")
+ FileFactory.getCarbonFile(tablePath, fileType)
+ .renameForce(CarbonTablePath.getNewTablePath(tablePath, oldCarbonTable.getTableName))
val absoluteTableIdentifier = AbsoluteTableIdentifier.from(
newTablePath,
newCarbonTableIdentifier)
@@ -233,9 +221,7 @@ object AlterTableUtil {
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath,
- carbonTable.getCarbonTableIdentifier)
- val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -260,9 +246,7 @@ object AlterTableUtil {
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath,
- carbonTable.getCarbonTableIdentifier)
- val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -293,9 +277,7 @@ object AlterTableUtil {
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath,
- carbonTable.getCarbonTableIdentifier)
- val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -344,9 +326,7 @@ object AlterTableUtil {
carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
// get the latest carbon table
// read the latest schema file
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath,
- carbonTable.getCarbonTableIdentifier)
- val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
thriftTableInfo,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index aadee81..0bdef8a 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -856,9 +856,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
}
def getDataFiles(carbonTable: CarbonTable, segmentId: String): Array[CarbonFile] = {
- val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index 56c5747..71c5477 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -24,7 +24,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.util.TableOptionConstant
@@ -65,9 +65,7 @@ class AllDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll {
carbonLoadModel.setCsvHeaderColumns(
CommonUtil.getCsvHeaderColumns(carbonLoadModel, FileFactory.getConfiguration))
// Create table and metadata folders if not exist
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(table.getTablePath, table.getCarbonTableIdentifier)
- val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
+ val metadataDirectoryPath = CarbonTablePath.getMetadataPath(table.getTablePath)
val fileType = FileFactory.getFileType(metadataDirectoryPath)
if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
FileFactory.mkdirs(metadataDirectoryPath, fileType)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
index f9519f8..a465251 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.test.TestQueryExecutor
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.util.CarbonLoaderUtil
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index a1b39d8..c0e1781 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -26,7 +26,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.util.TableOptionConstant
@@ -179,9 +179,7 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
CommonUtil.getCsvHeaderColumns(carbonLoadModel, FileFactory.getConfiguration))
carbonLoadModel.setMaxColumns("100")
// Create table and metadata folders if not exist
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(table.getTablePath, table.getCarbonTableIdentifier)
- val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
+ val metadataDirectoryPath = CarbonTablePath.getMetadataPath(table.getTablePath)
val fileType = FileFactory.getFileType(metadataDirectoryPath)
if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
FileFactory.mkdirs(metadataDirectoryPath, fileType)
[02/50] [abbrv] carbondata git commit: [CARBONDATA-1796] While
submiting new job, pass credentials in jobConf object
Posted by ra...@apache.org.
[CARBONDATA-1796] While submiting new job, pass credentials in jobConf object
This closes #1855
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b421c246
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b421c246
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b421c246
Branch: refs/heads/fgdatamap
Commit: b421c24689b18429f13fd150ab4dd422c61ca622
Parents: 2081fba
Author: akashrn5 <ak...@gmail.com>
Authored: Wed Jan 24 14:26:28 2018 +0530
Committer: QiangCai <qi...@qq.com>
Committed: Tue Jan 30 18:52:25 2018 +0800
----------------------------------------------------------------------
.../spark/rdd/CarbonIUDMergerRDD.scala | 3 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 2 ++
.../carbondata/spark/rdd/CarbonScanRDD.scala | 7 +++-
.../scala/org/apache/spark/util/SparkUtil.scala | 35 ++------------------
.../spark/rdd/CarbonDataRDDFactory.scala | 5 +--
5 files changed, 15 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
index e8180cd..4378c15 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
@@ -25,9 +25,9 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.{Partition, SparkContext}
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.execution.command.CarbonMergerMapping
-import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
@@ -57,6 +57,7 @@ class CarbonIUDMergerRDD[K, V](
tablePath, new CarbonTableIdentifier(databaseName, factTableName, tableId)
)
val jobConf: JobConf = new JobConf(new Configuration)
+ SparkHadoopUtil.get.addCredentials(jobConf)
val job: Job = new Job(jobConf)
val format = CarbonInputFormatUtil.createCarbonInputFormat(absoluteTableIdentifier, job)
val defaultParallelism = sparkContext.defaultParallelism
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 8d7b044..c482a92 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.Job
import org.apache.spark._
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.execution.command.{CarbonMergerMapping, NodeInfo}
import org.apache.spark.sql.hive.DistributionUtil
import org.apache.spark.sql.util.CarbonException
@@ -276,6 +277,7 @@ class CarbonMergerRDD[K, V](
val updateStatusManager: SegmentUpdateStatusManager = new SegmentUpdateStatusManager(
absoluteTableIdentifier)
val jobConf: JobConf = new JobConf(new Configuration)
+ SparkHadoopUtil.get.addCredentials(jobConf)
val job: Job = new Job(jobConf)
val format = CarbonInputFormatUtil.createCarbonInputFormat(absoluteTableIdentifier, job)
CarbonTableInputFormat.setPartitionsToPrune(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index f2c3060..49c0225 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -26,9 +26,11 @@ import scala.collection.mutable.ArrayBuffer
import scala.util.Random
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce._
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
import org.apache.spark._
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.hive.DistributionUtil
import org.apache.spark.sql.SparkSession
@@ -81,7 +83,10 @@ class CarbonScanRDD(
@transient val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
override def getPartitions: Array[Partition] = {
- val job = Job.getInstance(new Configuration())
+ val conf = new Configuration()
+ val jobConf = new JobConf(conf)
+ SparkHadoopUtil.get.addCredentials(jobConf)
+ val job = Job.getInstance(jobConf)
val format = prepareInputFormatForDriver(job.getConfiguration)
// initialise query_id for job
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
index 9c37640..4635fc7 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
@@ -19,8 +19,10 @@ package org.apache.spark.util
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.{LongWritable, Text}
+import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
import org.apache.spark.{SparkContext, TaskContext}
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.rdd.{NewHadoopPartition, NewHadoopRDD}
import org.apache.carbondata.processing.loading.csvinput.BlockDetails
@@ -37,37 +39,4 @@ object SparkUtil {
}
}
- /**
- * get file splits,return Array[BlockDetails], if file path is empty,then return empty Array
- *
- */
- def getSplits(path: String, sc: SparkContext): Array[BlockDetails] = {
- val filePath = FileUtils.getPaths(path)
- if (filePath == null || filePath.isEmpty) {
- // return a empty block details
- Array[BlockDetails]()
- } else {
- // clone the hadoop configuration
- val hadoopConfiguration = new Configuration(sc.hadoopConfiguration)
- // set folder or file
- hadoopConfiguration.set(FileInputFormat.INPUT_DIR, filePath)
- hadoopConfiguration.set(FileInputFormat.INPUT_DIR_RECURSIVE, "true")
- val newHadoopRDD = new NewHadoopRDD[LongWritable, Text](
- sc,
- classOf[org.apache.hadoop.mapreduce.lib.input.TextInputFormat],
- classOf[LongWritable],
- classOf[Text],
- hadoopConfiguration)
- val splits: Array[FileSplit] = newHadoopRDD.getPartitions.map { part =>
- part.asInstanceOf[NewHadoopPartition].serializableHadoopSplit.value.asInstanceOf[FileSplit]
- }
- splits.map { block =>
- new BlockDetails(block.getPath,
- block.getStart,
- block.getLength,
- block.getLocations
- )
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 809c8ff..8212e85 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -1030,9 +1030,10 @@ object CarbonDataRDDFactory {
org.apache.hadoop.io.compress.BZip2Codec""".stripMargin)
CommonUtil.configSplitMaxSize(sqlContext.sparkContext, filePaths, hadoopConf)
-
+ val jobConf = new JobConf(hadoopConf)
+ SparkHadoopUtil.get.addCredentials(jobConf)
val inputFormat = new org.apache.hadoop.mapreduce.lib.input.TextInputFormat
- val jobContext = new Job(hadoopConf)
+ val jobContext = new Job(jobConf)
val rawSplits = inputFormat.getSplits(jobContext).toArray
val blockList = rawSplits.map { inputSplit =>
val fileSplit = inputSplit.asInstanceOf[FileSplit]
[22/50] [abbrv] carbondata git commit: [CARBONDATA-2089]SQL exception
is masked due to assert(false) inside try catch and exception block always
asserting true
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3dff273b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala
index dab6e41..c57bd04 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala
@@ -21,9 +21,9 @@ package org.apache.carbondata.cluster.sdv.generated
import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util._
import org.scalatest.BeforeAndAfterAll
-
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.test.TestQueryExecutor
/**
* Test Class for singlepassTestCase to verify all scenerios
@@ -55,80 +55,51 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
//To check data loading from CSV with incomplete data
test("Loading-004-01-01-01_001-TC_003", Include) {
- try {
+ intercept[Exception] {
sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE if not exists uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_incomplete.csv' INTO TABLE uniqdata OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '"','SINGLE_PASS'='TRUE', 'FILEHEADER'= 'imei,deviceInformationId,AMSize,channelsId,ActiveCountry,Activecity,gamePointId,productionDate,deliveryDate,deliverycharge')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check data loading from CSV with bad records
test("Loading-004-01-01-01_001-TC_004", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_badrec.csv' INTO TABLE uniqdata OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '"','SINGLE_PASS'='TRUE', 'FILEHEADER'= 'imei,deviceInformationId,AMSize,channelsId,ActiveCountry,Activecity,gamePointId,productionDate,deliveryDate,deliverycharge')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check data loading from CSV with no data
test("Loading-004-01-01-01_001-TC_005", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_nodata.csv' INTO TABLE uniqdata OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '"','SINGLE_PASS'='TRUE', 'FILEHEADER'= 'imei,deviceInformationId,AMSize,channelsId,ActiveCountry,Activecity,gamePointId,productionDate,deliveryDate,deliverycharge')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check data loading from CSV with incomplete data
test("Loading-004-01-01-01_001-TC_006", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_incomplete.csv' INTO TABLE uniqdata OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '"','SINGLE_PASS'='FALSE', 'FILEHEADER'= 'imei,deviceInformationId,AMSize,channelsId,ActiveCountry,Activecity,gamePointId,productionDate,deliveryDate,deliverycharge')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check data loading from CSV with wrong data
test("Loading-004-01-01-01_001-TC_007", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_incomplete.csv' INTO TABLE uniqdata OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '"','SINGLE_PASS'='FALSE', 'FILEHEADER'= 'imei,deviceInformationId,AMSize,channelsId,ActiveCountry,Activecity,gamePointId,productionDate,deliveryDate,deliverycharge')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
//To check data loading from CSV with no data and 'SINGLEPASS' = 'FALSE'
test("Loading-004-01-01-01_001-TC_008", Include) {
- try {
-
+ intercept[Exception] {
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_nodata.csv.csv' INTO TABLE uniqdata OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '"','SINGLE_PASS'='FALSE', 'FILEHEADER'= 'imei,deviceInformationId,AMSize,channelsId,ActiveCountry,Activecity,gamePointId,productionDate,deliveryDate,deliverycharge')""").collect
- assert(false)
- } catch {
- case _ => assert(true)
}
-
}
@@ -555,22 +526,35 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
//Verifying load data with single Pass true and BAD_RECORDS_ACTION= ='FAIL
test("Loading-004-01-01-01_001-TC_067", Include) {
sql(s"""drop table if exists uniqdata""").collect
- try {
-
- sql(s"""CREATE TABLE if not exists uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""")
+ intercept[Exception] {
+ sql(s"""
+ | CREATE TABLE uniqdata(
+ | shortField SHORT,
+ | booleanField BOOLEAN,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | decimalField DECIMAL(18,2),
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>,
+ | booleanField2 BOOLEAN
+ | )
+ | STORED BY 'carbondata'
+ """.stripMargin)
.collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/data/2000_UniqData.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='FAIL','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','SINGLE_Pass'='true')""")
+ sql(
+ s"""LOAD DATA INPATH '${TestQueryExecutor
+ .integrationPath}/spark2/src/test/resources/bool/supportBooleanBadRecords.csv' into table uniqdata OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='FAIL','FILEHEADER'='shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData,booleanField2','SINGLE_Pass'='true')""".stripMargin)
.collect
checkAnswer(
s"""select count(*) from uniqdata""",
Seq(Row(2013)),
"singlepassTestCase_Loading-004-01-01-01_001-TC_067")
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table uniqdata""").collect
}
@@ -578,6 +562,7 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
//Verifying load data with single Pass true and BAD_RECORDS_ACTION= ='REDIRECT'
test("Loading-004-01-01-01_001-TC_071", Include) {
+ sql(s"""drop table if exists uniqdata""").collect
sql(s"""CREATE TABLE if not exists uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
@@ -717,7 +702,7 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
//Verifying load data with single pass=false and column dictionary path
test("Loading-004-01-01-01_001-TC_084", Include) {
dropTable("uniqdata")
- try {
+ intercept[Exception] {
sql(s"""CREATE TABLE if not exists uniqdata (CUST_ID int,CUST_NAME String, DOB timestamp) STORED BY 'org.apache.carbondata.format'""")
.collect
@@ -727,9 +712,6 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
s"""select count(*) from uniqdata""",
Seq(Row(10)),
"singlepassTestCase_Loading-004-01-01-01_001-TC_084")
- assert(false)
- } catch {
- case _ => assert(true)
}
sql(s"""drop table uniqdata""").collect
}
[37/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
index 9391ebd..44f7c07 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -16,20 +16,10 @@
*/
package org.apache.carbondata.core.scan.filter.executer;
-import java.util.ArrayList;
import java.util.BitSet;
-import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.util.ByteUtil;
/**
* It checks if filter is required on given block and if required, it does
@@ -38,12 +28,6 @@ import org.apache.carbondata.core.util.ByteUtil;
public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl {
/**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ExcludeColGroupFilterExecuterImpl.class.getName());
-
- /**
* @param dimColResolvedFilterInfo
* @param segmentProperties
*/
@@ -53,54 +37,6 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl
}
/**
- * It fills BitSet with row index which matches filter key
- */
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- bitSet.flip(0, numerOfRows);
- try {
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- byte[] filterVal = filterValues[i];
- for (int rowId = 0; rowId < numerOfRows; rowId++) {
- byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
- bitSet.flip(rowId);
- }
- }
- }
-
- } catch (Exception e) {
- LOGGER.error(e);
- }
-
- return bitSet;
- }
-
- /**
- * It is required for extracting column data from columngroup chunk
- *
- * @return
- * @throws KeyGenException
- */
- private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
- int colGrpId = getColumnGroupId(dimColEvaluatorInfo.getColumnIndex());
- KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
- mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(), colGrpId));
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-
- /**
* Check if scan is required on given block based on min and max value
*/
public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
@@ -109,25 +45,4 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl
return bitSet;
}
- private int getMdkeyOrdinal(int ordinal, int colGrpId) {
- return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
- }
-
- private int getColumnGroupId(int ordinal) {
- int[][] columnGroups = segmentProperties.getColumnGroups();
- int colGrpId = -1;
- for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
- colGrpId++;
- if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
- }
- }
- }
- return colGrpId;
- }
-
- public KeyGenerator getKeyGenerator(int colGrpId) {
- return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 465bee6..5974666 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -30,7 +30,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -40,13 +40,13 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator;
public class ExcludeFilterExecuterImpl implements FilterExecuter {
- protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
- protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
- protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
- protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
+ private DimColumnResolvedFilterInfo dimColEvaluatorInfo;
+ private DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+ private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+ private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
protected SegmentProperties segmentProperties;
- protected boolean isDimensionPresentInCurrentBlock = false;
- protected boolean isMeasurePresentInCurrentBlock = false;
+ private boolean isDimensionPresentInCurrentBlock = false;
+ private boolean isMeasurePresentInCurrentBlock = false;
private SerializableComparator comparator;
/**
* is dimension column data is natural sorted
@@ -82,48 +82,50 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
if (isDimensionPresentInCurrentBlock) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk dimensionRawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
- DimensionColumnDataChunk[] dimensionColumnDataChunks =
- dimensionRawColumnChunk.convertToDimColDataChunks();
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ DimensionColumnPage[] dimensionColumnPages =
+ dimensionRawColumnChunk.decodeAllColumnPages();
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
- for (int i = 0; i < dimensionColumnDataChunks.length; i++) {
- BitSet bitSet = getFilteredIndexes(dimensionColumnDataChunks[i],
+ for (int i = 0; i < dimensionColumnPages.length; i++) {
+ BitSet bitSet = getFilteredIndexes(dimensionColumnPages[i],
dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i);
+ rawBlockletColumnChunks.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
MeasureRawColumnChunk measureRawColumnChunk =
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
ColumnPage[] ColumnPages =
- measureRawColumnChunk.convertToColumnPage();
+ measureRawColumnChunk.decodeAllColumnPages();
BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount());
DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
for (int i = 0; i < ColumnPages.length; i++) {
BitSet bitSet =
getFilteredIndexesForMeasure(
- measureRawColumnChunk.convertToColumnPage(i),
+ measureRawColumnChunk.decodeColumnPage(i),
measureRawColumnChunk.getRowCount()[i],
useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(),
+ rawBlockletColumnChunks.getBitSetGroup(),
i,
msrType);
bitSetGroup.setBitSet(bitSet, i);
@@ -133,7 +135,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return null;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
if (isDimensionPresentInCurrentBlock) {
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
byte[] col = (byte[])value.getVal(dimColEvaluatorInfo.getDimension().getOrdinal());
@@ -275,43 +278,41 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
/**
* Below method will be used to apply filter on dimension column
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numberOfRows
* @param useBitsetPipeLine
* @param prvBitSetGroup
* @param pageNumber
* @return filtered indexes bitset
*/
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
// check whether applying filtered based on previous bitset will be optimal
if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
dimColumnExecuterInfo.getExcludeFilterKeys().length)) {
- return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
- numberOfRows);
+ return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber);
} else {
- return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+ return getFilteredIndexes(dimensionColumnPage, numberOfRows);
}
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows);
}
/**
* Below method will be used to apply filter based on previous filtered bitset
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param prvBitSetGroup
* @param pageNumber
- * @param numberOfRows
* @return filtered indexes bitset
*/
- private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
- BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
+ private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
+ BitSetGroup prvBitSetGroup, int pageNumber) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
BitSet bitSet = new BitSet();
bitSet.or(prvPageBitSet);
@@ -319,11 +320,11 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
int compareResult = 0;
// if dimension data was natural sorted then get the index from previous bitset
// and use the same in next column data, otherwise use the inverted index reverse
- if (!dimensionColumnDataChunk.isExplicitSorted()) {
+ if (!dimensionColumnPage.isExplicitSorted()) {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index);
if (compareResult != 0) {
bitSet.set(index);
} else {
@@ -336,8 +337,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
- dimensionColumnDataChunk.getInvertedReverseIndex(index));
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1,
+ dimensionColumnPage.getInvertedReverseIndex(index));
if (compareResult != 0) {
bitSet.set(index);
} else {
@@ -351,7 +352,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
int startIndex = 0;
@@ -361,10 +362,10 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
- bitSet.flip(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.flip(dimensionColumnPage.getInvertedIndex(j));
}
if (rangeIndex[1] >= 0) {
startIndex = rangeIndex[1] + 1;
@@ -373,7 +374,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
@@ -390,7 +391,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
bitSet.flip(j);
@@ -403,14 +404,14 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
if (filterValues.length > 1) {
for (int i = 0; i < numerOfRows; i++) {
int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
- dimensionColumnDataChunk.getChunkData(i));
+ dimensionColumnPage.getChunkData(i));
if (index >= 0) {
bitSet.flip(i);
}
}
} else {
for (int j = 0; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[0]) == 0) {
+ if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) {
bitSet.flip(j);
}
}
@@ -419,26 +420,30 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
bitSet.flip(0, 1);
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
index 85891dc..aac5e63 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
@@ -21,7 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public interface FilterExecuter {
@@ -32,8 +32,8 @@ public interface FilterExecuter {
* @return
* @throws FilterUnsupportedException
*/
- BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException;
+ BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException;
boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException;
@@ -50,7 +50,7 @@ public interface FilterExecuter {
/**
* It just reads necessary block for filter executor, it does not uncompress the data.
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
*/
- void readBlocks(BlocksChunkHolder blockChunkHolder)throws IOException;
+ void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks)throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
index 9c4c7ba..b683fd6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
@@ -23,7 +23,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -41,26 +41,31 @@ public class ImplicitIncludeFilterExecutorImpl
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeline)
- throws FilterUnsupportedException {
- BitSetGroup bitSetGroup = new BitSetGroup(blockChunkHolder.getDataBlock().numberOfPages());
- for (int i = 0; i < blockChunkHolder.getDataBlock().numberOfPages(); i++) {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeline) {
+ BitSetGroup bitSetGroup = new BitSetGroup(
+ rawBlockletColumnChunks.getDataBlock().numberOfPages());
+ for (int i = 0; i < rawBlockletColumnChunks.getDataBlock().numberOfPages(); i++) {
bitSetGroup.setBitSet(
- setBitSetForCompleteDimensionData(blockChunkHolder.getDataBlock().getPageRowCount(i)), i);
+ setBitSetForCompleteDimensionData(
+ rawBlockletColumnChunks.getDataBlock().getPageRowCount(i)), i);
}
return bitSetGroup;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return false;
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
return null;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
index 1cbc0bc..e4da26f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
@@ -24,14 +24,14 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
@@ -59,7 +59,7 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
/**
* It fills BitSet with row index which matches filter key
*/
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
@@ -70,7 +70,7 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
byte[] filterVal = filterValues[i];
for (int rowId = 0; rowId < numerOfRows; rowId++) {
byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
+ dimensionColumnPage.fillRawData(rowId, 0, colData, keyStructureInfo);
if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
bitSet.set(rowId);
}
@@ -85,20 +85,21 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk dimensionRawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
if (dimensionRawColumnChunk.getMaxValues() != null) {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
dimensionRawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -133,10 +134,10 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
BitSet bitSet = new BitSet(1);
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex);
int[] cols = getAllColumns(columnIndex);
- byte[] maxValue = getMinMaxData(cols, blkMaxVal[blockIndex], columnIndex);
- byte[] minValue = getMinMaxData(cols, blkMinVal[blockIndex], columnIndex);
+ byte[] maxValue = getMinMaxData(cols, blkMaxVal[chunkIndex], columnIndex);
+ byte[] minValue = getMinMaxData(cols, blkMinVal[chunkIndex], columnIndex);
boolean isScanRequired = false;
for (int k = 0; k < filterValues.length; k++) {
// filter value should be in range of max and min value i.e
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index fe1421c..05328f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -30,7 +30,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -41,12 +41,12 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator;
public class IncludeFilterExecuterImpl implements FilterExecuter {
protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
- protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
- protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
- protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
+ DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+ private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+ private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
protected SegmentProperties segmentProperties;
- protected boolean isDimensionPresentInCurrentBlock = false;
- protected boolean isMeasurePresentInCurrentBlock = false;
+ private boolean isDimensionPresentInCurrentBlock = false;
+ private boolean isMeasurePresentInCurrentBlock = false;
protected SerializableComparator comparator;
/**
* is dimension column data is natural sorted
@@ -86,44 +86,46 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk dimensionRawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
if (dimensionRawColumnChunk.getMaxValues() != null) {
if (isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys())) {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i);
+ rawBlockletColumnChunks.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i);
+ rawBlockletColumnChunks.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
MeasureRawColumnChunk measureRawColumnChunk =
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount());
DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
for (int i = 0; i < measureRawColumnChunk.getPagesCount(); i++) {
@@ -132,16 +134,16 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
measureRawColumnChunk.getMinValues()[i], msrColumnExecutorInfo.getFilterKeys(),
msrColumnEvaluatorInfo.getType())) {
BitSet bitSet =
- getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasure(measureRawColumnChunk.decodeColumnPage(i),
measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i, msrType);
+ rawBlockletColumnChunks.getBitSetGroup(), i, msrType);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasure(measureRawColumnChunk.decodeColumnPage(i),
measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i, msrType);
+ rawBlockletColumnChunks.getBitSetGroup(), i, msrType);
bitSetGroup.setBitSet(bitSet, i);
}
}
@@ -293,43 +295,43 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
/**
* Below method will be used to apply filter on dimension column
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numberOfRows
* @param useBitsetPipeLine
* @param prvBitSetGroup
* @param pageNumber
* @return filtered indexes bitset
*/
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
// check whether previous indexes can be optimal to apply filter on dimension column
if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
dimColumnExecuterInfo.getFilterKeys().length)) {
- return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
+ return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber,
numberOfRows);
} else {
- return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+ return getFilteredIndexes(dimensionColumnPage, numberOfRows);
}
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows);
}
/**
* Below method will be used to apply filter on dimension
* column based on previous filtered indexes
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param prvBitSetGroup
* @param pageNumber
* @param numberOfRows
* @return filtered bitset
*/
- private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
BitSet bitSet = new BitSet(numberOfRows);
@@ -337,11 +339,11 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
int compareResult = 0;
// if dimension data was natural sorted then get the index from previous bitset
// and use the same in next column data, otherwise use the inverted index reverse
- if (!dimensionColumnDataChunk.isExplicitSorted()) {
+ if (!dimensionColumnPage.isExplicitSorted()) {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index);
if (compareResult == 0) {
bitSet.set(index);
}
@@ -350,8 +352,8 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
- dimensionColumnDataChunk.getInvertedReverseIndex(index));
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1,
+ dimensionColumnPage.getInvertedReverseIndex(index));
if (compareResult == 0) {
bitSet.set(index);
}
@@ -360,7 +362,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int startIndex = 0;
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
@@ -369,10 +371,10 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
}
if (rangeIndex[1] >= 0) {
startIndex = rangeIndex[1] + 1;
@@ -381,7 +383,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
@@ -394,7 +396,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
bitSet.set(j);
@@ -407,14 +409,14 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
if (filterValues.length > 1) {
for (int i = 0; i < numerOfRows; i++) {
int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
- dimensionColumnDataChunk.getChunkData(i));
+ dimensionColumnPage.getChunkData(i));
if (index >= 0) {
bitSet.set(i);
}
}
} else {
for (int j = 0; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[0]) == 0) {
+ if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) {
bitSet.set(j);
}
}
@@ -423,26 +425,25 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
+ @Override
public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
BitSet bitSet = new BitSet(1);
byte[][] filterValues = null;
int columnIndex = 0;
- int blockIndex = 0;
+ int chunkIndex = 0;
boolean isScanRequired = false;
if (isDimensionPresentInCurrentBlock) {
filterValues = dimColumnExecuterInfo.getFilterKeys();
columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
- isScanRequired =
- isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], filterValues);
-
+ chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex);
+ isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], filterValues);
} else if (isMeasurePresentInCurrentBlock) {
columnIndex = msrColumnEvaluatorInfo.getColumnIndex();
- blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(columnIndex) + segmentProperties
- .getLastDimensionColOrdinal();
- isScanRequired = isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex],
+ chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(columnIndex) +
+ segmentProperties.getLastDimensionColOrdinal();
+ isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex],
msrColumnExecutorInfo.getFilterKeys(),
msrColumnEvaluatorInfo.getType());
}
@@ -492,20 +493,23 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return false;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- } else if (isMeasurePresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ } else if (isMeasurePresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
index 87273bb..9f0afb5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
@@ -21,7 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class OrFilterExecuterImpl implements FilterExecuter {
@@ -35,30 +35,33 @@ public class OrFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, false);
- BitSetGroup rightFilters = rightExecuter.applyFilter(blockChunkHolder, false);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ BitSetGroup leftFilters = leftExecuter.applyFilter(rawBlockletColumnChunks, false);
+ BitSetGroup rightFilters = rightExecuter.applyFilter(rawBlockletColumnChunks, false);
leftFilters.or(rightFilters);
- blockChunkHolder.setBitSetGroup(leftFilters);
+ rawBlockletColumnChunks.setBitSetGroup(leftFilters);
return leftFilters;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return leftExecuter.applyFilter(value, dimOrdinalMax) ||
rightExecuter.applyFilter(value, dimOrdinalMax);
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet leftFilters = leftExecuter.isScanRequired(blockMaxValue, blockMinValue);
BitSet rightFilters = rightExecuter.isScanRequired(blockMaxValue, blockMinValue);
leftFilters.or(rightFilters);
return leftFilters;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- leftExecuter.readBlocks(blockChunkHolder);
- rightExecuter.readBlocks(blockChunkHolder);
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ leftExecuter.readColumnChunks(rawBlockletColumnChunks);
+ rightExecuter.readColumnChunks(rawBlockletColumnChunks);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index ee373c5..4a9716c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -22,11 +22,10 @@ import java.util.BitSet;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -38,8 +37,7 @@ import org.apache.carbondata.core.scan.expression.conditional.LessThanExpression
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -59,25 +57,23 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* it has index at which given dimension is stored in file
*/
- protected int dimensionBlocksIndex;
+ private int dimensionChunkIndex;
/**
* flag to check whether the filter dimension is present in current block list of dimensions.
* Applicable for restructure scenarios
*/
- protected boolean isDimensionPresentInCurrentBlock;
- boolean lessThanExp;
- boolean lessThanEqualExp;
- boolean greaterThanExp;
- boolean greaterThanEqualExp;
- boolean startBlockMinIsDefaultStart;
- boolean endBlockMaxisDefaultEnd;
- boolean isRangeFullyCoverBlock;
+ private boolean isDimensionPresentInCurrentBlock;
+ private boolean lessThanExp;
+ private boolean lessThanEqualExp;
+ private boolean greaterThanExp;
+ private boolean greaterThanEqualExp;
+ private boolean startBlockMinIsDefaultStart;
+ private boolean endBlockMaxisDefaultEnd;
+ private boolean isRangeFullyCoverBlock;
public RangeValueFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
- MeasureColumnResolvedFilterInfo msrColEvaluatorInfo, Expression exp,
- AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
- SegmentProperties segmentProperties) {
+ Expression exp, byte[][] filterRangeValues, SegmentProperties segmentProperties) {
this.dimColEvaluatorInfo = dimColEvaluatorInfo;
this.exp = exp;
@@ -90,7 +86,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
startBlockMinIsDefaultStart = false;
endBlockMaxisDefaultEnd = false;
isRangeFullyCoverBlock = false;
- initDimensionBlockIndexes();
+ initDimensionChunkIndexes();
ifDefaultValueMatchesFilter();
}
@@ -99,13 +95,13 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* This method will initialize the dimension info for the current block to be
* used for filtering the data
*/
- private void initDimensionBlockIndexes() {
+ private void initDimensionChunkIndexes() {
// find the dimension in the current block dimensions list
CarbonDimension dimensionFromCurrentBlock =
segmentProperties.getDimensionFromCurrentBlock(dimColEvaluatorInfo.getDimension());
if (null != dimensionFromCurrentBlock) {
dimColEvaluatorInfo.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
- this.dimensionBlocksIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ this.dimensionChunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimensionFromCurrentBlock.getOrdinal());
isDimensionPresentInCurrentBlock = true;
}
@@ -135,14 +131,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* Method to apply the filter.
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
* @return
* @throws FilterUnsupportedException
* @throws IOException
*/
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- return applyNoAndDirectFilter(blockChunkHolder);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ return applyNoAndDirectFilter(rawBlockletColumnChunks);
}
/**
@@ -243,32 +239,32 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Filter Min <-----------------------------------------------> Filter Max
if (isDimensionPresentInCurrentBlock) {
- if (((lessThanExp == true) && (
+ if (((lessThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) >= 0)) || (
- (lessThanEqualExp == true) && (
+ (lessThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) > 0)) || (
- (greaterThanExp == true) && (
+ (greaterThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) >= 0)) || (
- (greaterThanEqualExp == true) && (
+ (greaterThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) > 0))) {
// completely out of block boundary
isScanRequired = false;
} else {
- if (((greaterThanExp == true) && (
+ if (((greaterThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) > 0)) || (
- (greaterThanEqualExp == true) && (
+ (greaterThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) >= 0))) {
startBlockMinIsDefaultStart = true;
}
- if (((lessThanExp == true) && (
+ if (((lessThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) > 0)) || (
- (lessThanEqualExp == true) && (
+ (lessThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) >= 0))) {
endBlockMaxisDefaultEnd = true;
}
- if (startBlockMinIsDefaultStart == true && endBlockMaxisDefaultEnd == true) {
+ if (startBlockMinIsDefaultStart && endBlockMaxisDefaultEnd) {
isRangeFullyCoverBlock = true;
}
}
@@ -284,7 +280,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* @param blockMinValue
* @return
*/
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
byte[][] filterValues = this.filterRangesValues;
int columnIndex = this.dimColEvaluatorInfo.getColumnIndex();
@@ -300,50 +297,49 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* Method to apply the Range Filter.
* @param blockChunkHolder
* @return
- * @throws FilterUnsupportedException
* @throws IOException
*/
- public BitSetGroup applyNoAndDirectFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
+ private BitSetGroup applyNoAndDirectFilter(RawBlockletColumnChunks blockChunkHolder)
+ throws IOException {
// In case of Alter Table Add and Delete Columns the isDimensionPresentInCurrentBlock can be
// false, in that scenario the default values of the column should be shown.
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
- return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
- numberOfRows, true);
+ int numberOfRows = blockChunkHolder.getDataBlock().numRows();
+ return FilterUtil.createBitSetGroupWithDefaultValue(
+ blockChunkHolder.getDataBlock().numberOfPages(), numberOfRows, true);
}
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex]) {
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex] =
+ blockChunkHolder.getDataBlock().readDimensionChunk(
+ blockChunkHolder.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], rawColumnChunk.getMaxValues()[i],
this.filterRangesValues)) {
- if (isRangeFullyCoverBlock == true) {
+ if (isRangeFullyCoverBlock) {
// Set all the bits in this case as filter Min Max values cover the whole block.
BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -351,12 +347,12 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
return bitSetGroup;
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows);
}
/**
@@ -365,12 +361,12 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
int startIndex = 0;
@@ -381,13 +377,13 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// For Range expression we expect two values. The First is the Min Value and Second is the
// Max value.
// Get the Min Value
- if (startBlockMinIsDefaultStart == false) {
+ if (!startBlockMinIsDefaultStart) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[0], greaterThanExp);
- if (greaterThanExp == true && start >= 0) {
+ if (greaterThanExp && start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[0],
numerOfRows);
}
@@ -399,8 +395,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk
- .getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))) > 0) {
+ if ((ByteUtil.compare(filterValues[0], dimensionColumnPage
+ .getChunkData(dimensionColumnPage.getInvertedIndex(start)))) > 0) {
start = start + 1;
}
}
@@ -411,14 +407,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
// Get the Max value
- if (endBlockMaxisDefaultEnd == false) {
+ if (!endBlockMaxisDefaultEnd) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[1], lessThanEqualExp);
- if (lessThanExp == true && start >= 0) {
+ if (lessThanExp && start >= 0) {
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[1]);
}
if (start < 0) {
@@ -429,7 +425,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// In case the start is less than 0, then positive value of start is pointing to the next
// value of the searched key. So move to the previous one.
if ((ByteUtil.compare(filterValues[1],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
< 0)) {
start = start - 1;
}
@@ -440,18 +436,18 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
for (int j = startMin; j <= endMax; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
}
// Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on
// matching row.
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet);
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ updateForNoDictionaryColumn(startMin, endMax, dimensionColumnPage, bitSet);
}
return bitSet;
}
- private void updateForNoDictionaryColumn(int start, int end, DimensionColumnDataChunk dataChunk,
+ private void updateForNoDictionaryColumn(int start, int end, DimensionColumnPage dataChunk,
BitSet bitset) {
for (int j = start; j <= end; j++) {
if (dataChunk.compareTo(j, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY) == 0
@@ -467,31 +463,31 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* be called if the column is sorted default so column index
* mapping will be present for accesaing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
- // if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ // if (dimensionColumnPage instanceof FixedLengthDimensionColumnPage) {
byte[][] filterValues = this.filterRangesValues;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
+ if (dimensionColumnPage.isExplicitSorted()) {
int start = 0;
int startMin = 0;
int endMax = 0;
int startIndex = 0;
// For Range expression we expect two values. The First is the Min Value and Second is the
// Max value.
- if (startBlockMinIsDefaultStart == false) {
+ if (!startBlockMinIsDefaultStart) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[0], greaterThanExp);
- if (greaterThanExp == true && start >= 0) {
+ if (greaterThanExp && start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[0],
numerOfRows);
}
@@ -503,7 +499,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk.getChunkData(start)))
+ if ((ByteUtil.compare(filterValues[0], dimensionColumnPage.getChunkData(start)))
> 0) {
start = start + 1;
}
@@ -513,14 +509,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
startMin = startIndex;
}
- if (endBlockMaxisDefaultEnd == false) {
+ if (!endBlockMaxisDefaultEnd) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[1], lessThanEqualExp);
- if (lessThanExp == true && start >= 0) {
+ if (lessThanExp && start >= 0) {
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[1]);
}
if (start < 0) {
@@ -530,7 +526,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
// In case the start is less than 0, then positive value of start is pointing to the next
// value of the searched key. So move to the previous one.
- if ((ByteUtil.compare(filterValues[1], dimensionColumnDataChunk.getChunkData(start))
+ if ((ByteUtil.compare(filterValues[1], dimensionColumnPage.getChunkData(start))
< 0)) {
start = start - 1;
}
@@ -546,8 +542,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on
// matching row.
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet);
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ updateForNoDictionaryColumn(startMin, endMax, dimensionColumnPage, bitSet);
}
} else {
byte[] defaultValue = null;
@@ -556,7 +552,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
.getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType());
int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
- segmentProperties.getDimensions().get(dimensionBlocksIndex);
+ segmentProperties.getDimensions().get(dimensionChunkIndex);
if (currentBlockDimension.isSortColumn()) {
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
this.segmentProperties.getSortColumnsGenerator());
@@ -572,55 +568,37 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
// evaluate result for lower range value first and then perform and operation in the
// upper range value in order to compute the final result
- bitSet = evaluateGreaterThanFilterForUnsortedColumn(dimensionColumnDataChunk, filterValues[0],
+ bitSet = evaluateGreaterThanFilterForUnsortedColumn(dimensionColumnPage, filterValues[0],
numerOfRows);
BitSet upperRangeBitSet =
- evaluateLessThanFilterForUnsortedColumn(dimensionColumnDataChunk, filterValues[1],
+ evaluateLessThanFilterForUnsortedColumn(dimensionColumnPage, filterValues[1],
numerOfRows);
bitSet.and(upperRangeBitSet);
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet, defaultValue);
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
}
return bitSet;
}
/**
- * This method will compare the selected data against null values and
- * flip the bitSet if any null value is found
- *
- * @param dimensionColumnDataChunk
- * @param bitSet
- */
- private void removeNullValues(DimensionColumnDataChunk dimensionColumnDataChunk, BitSet bitSet) {
- if (!bitSet.isEmpty()) {
- for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
- if (dimensionColumnDataChunk.compareTo(i, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY)
- == 0) {
- bitSet.flip(i);
- }
- }
- }
- }
-
- /**
* This method will evaluate the result for filter column based on the lower range value
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param filterValue
* @param numberOfRows
* @return
*/
private BitSet evaluateGreaterThanFilterForUnsortedColumn(
- DimensionColumnDataChunk dimensionColumnDataChunk, byte[] filterValue, int numberOfRows) {
+ DimensionColumnPage dimensionColumnPage, byte[] filterValue, int numberOfRows) {
BitSet bitSet = new BitSet(numberOfRows);
if (greaterThanExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) > 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) > 0)) {
bitSet.set(i);
}
}
} else if (greaterThanEqualExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) >= 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) >= 0)) {
bitSet.set(i);
}
}
@@ -631,23 +609,23 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* This method will evaluate the result for filter column based on the upper range value
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param filterValue
* @param numberOfRows
* @return
*/
private BitSet evaluateLessThanFilterForUnsortedColumn(
- DimensionColumnDataChunk dimensionColumnDataChunk, byte[] filterValue, int numberOfRows) {
+ DimensionColumnPage dimensionColumnPage, byte[] filterValue, int numberOfRows) {
BitSet bitSet = new BitSet(numberOfRows);
if (lessThanExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) < 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) < 0)) {
bitSet.set(i);
}
}
} else if (lessThanEqualExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) <= 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) <= 0)) {
bitSet.set(i);
}
}
@@ -657,16 +635,18 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* Method to read the blocks.
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
* @throws IOException
*/
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
index 5707eb4..d7bec7e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
@@ -24,7 +24,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorImpl {
@@ -34,7 +34,7 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm
* flag to check whether filter values contain the default value applied on the dimension column
* which does not exist in the current block
*/
- protected boolean isDefaultValuePresentInFilterValues;
+ private boolean isDefaultValuePresentInFilterValues;
public RestructureExcludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) {
@@ -48,26 +48,29 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, !isDefaultValuePresentInFilterValues);
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException {
throw new FilterUnsupportedException("Unsupported RestructureExcludeFilterExecutorImpl on row");
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
bitSet.flip(0, 1);
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
index 8bcc53f..c874fc5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
@@ -24,7 +24,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorImpl {
@@ -33,7 +33,7 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
* flag to check whether filter values contain the default value applied on the dimension column
* which does not exist in the current block
*/
- protected boolean isDefaultValuePresentInFilterValues;
+ private boolean isDefaultValuePresentInFilterValues;
public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) {
@@ -47,15 +47,16 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
- return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
- numberOfRows, isDefaultValuePresentInFilterValues);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
+ return FilterUtil.createBitSetGroupWithDefaultValue(
+ rawBlockletColumnChunks.getDataBlock().numberOfPages(),
+ numberOfRows, isDefaultValuePresentInFilterValues);
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException {
throw new FilterUnsupportedException("Unsupported RestructureIncludeFilterExecutorImpl on row");
}
@@ -66,7 +67,8 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) {
}
[45/50] [abbrv] carbondata git commit: [CARBONDATA-2025] Unify all
path construction through CarbonTablePath static method
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
index d0aedd4..c36c89d 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
@@ -56,7 +56,7 @@ public class ForwardDictionaryCacheTest extends AbstractDictionaryCacheTest {
this.carbonStorePath = props.getProperty("storePath", "carbonStore");
carbonTableIdentifier =
new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
- absoluteTableIdentifier =
+ identifier =
AbsoluteTableIdentifier.from(carbonStorePath + "/" + databaseName + "/" + tableName,
carbonTableIdentifier);
columnIdentifiers = new String[] { "name", "place" };
@@ -67,7 +67,7 @@ public class ForwardDictionaryCacheTest extends AbstractDictionaryCacheTest {
@After public void tearDown() throws Exception {
carbonTableIdentifier = null;
- absoluteTableIdentifier = null;
+ identifier = null;
forwardDictionaryCache = null;
deleteStorePath();
}
@@ -217,7 +217,7 @@ public class ForwardDictionaryCacheTest extends AbstractDictionaryCacheTest {
private void writeSortIndexFile(List<String> data, String columnId) throws IOException {
ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, null);
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
- new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier,
+ new DictionaryColumnUniqueIdentifier(identifier, columnIdentifier,
columnIdentifier.getDataType());
Map<String, Integer> dataToSurrogateKeyMap = new HashMap<>(data.size());
int surrogateKey = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
index 01cb3a9..d2bf2e3 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
@@ -58,7 +58,7 @@ public class ReverseDictionaryCacheTest extends AbstractDictionaryCacheTest {
this.carbonStorePath = props.getProperty("storePath", "carbonStore");
carbonTableIdentifier =
new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
- absoluteTableIdentifier = AbsoluteTableIdentifier.from(
+ identifier = AbsoluteTableIdentifier.from(
carbonStorePath + "/" + databaseName + "/" + tableName, carbonTableIdentifier);
columnIdentifiers = new String[] { "name", "place" };
deleteStorePath();
@@ -69,7 +69,7 @@ public class ReverseDictionaryCacheTest extends AbstractDictionaryCacheTest {
@After public void tearDown() throws Exception {
carbonTableIdentifier = null;
reverseDictionaryCache = null;
- absoluteTableIdentifier = null;
+ identifier = null;
deleteStorePath();
}
@@ -276,6 +276,6 @@ public class ReverseDictionaryCacheTest extends AbstractDictionaryCacheTest {
protected DictionaryColumnUniqueIdentifier createDictionaryColumnUniqueIdentifier(
String columnId) {
ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, DataTypes.DOUBLE);
- return new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier);
+ return new DictionaryColumnUniqueIdentifier(identifier, columnIdentifier);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
index d3c3bc3..ecabfd4 100644
--- a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
@@ -28,8 +28,6 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.service.CarbonCommonFactory;
-import org.apache.carbondata.core.service.PathService;
-import org.apache.carbondata.core.service.impl.PathFactory;
import mockit.Mock;
import mockit.MockUp;
@@ -58,12 +56,6 @@ public class CarbonDictionaryReaderImplTest {
}
@Test public void testRead() throws Exception {
- new MockUp<CarbonCommonFactory>() {
- @Mock public PathService getPathService() {
-
- return new PathFactory();
- }
- };
new MockUp<CarbonDictionaryMetadataReaderImpl>() {
@Mock public List<CarbonDictionaryColumnMetaChunk> read() throws IOException {
CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunks =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
index a1ccab3..4293536 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
@@ -39,21 +39,19 @@ public class CarbonFormatDirectoryStructureTest {
*/
@Test public void testTablePathStructure() throws IOException {
CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier("d1", "t1", UUID.randomUUID().toString());
- CarbonStorePath carbonStorePath = new CarbonStorePath(CARBON_STORE);
- AbsoluteTableIdentifier absoluteTableIdentifier =
+ AbsoluteTableIdentifier identifier =
AbsoluteTableIdentifier.from(CARBON_STORE + "/d1/t1", tableIdentifier);
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- assertTrue(carbonTablePath.getPath().replace("\\", "/").equals(CARBON_STORE + "/d1/t1"));
- assertTrue(carbonTablePath.getSchemaFilePath().replace("\\", "/").equals(CARBON_STORE + "/d1/t1/Metadata/schema"));
- assertTrue(carbonTablePath.getTableStatusFilePath().replace("\\", "/")
+ assertTrue(identifier.getTablePath().replace("\\", "/").equals(CARBON_STORE + "/d1/t1"));
+ assertTrue(CarbonTablePath.getSchemaFilePath(identifier.getTablePath()).replace("\\", "/").equals(CARBON_STORE + "/d1/t1/Metadata/schema"));
+ assertTrue(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()).replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Metadata/tablestatus"));
- assertTrue(carbonTablePath.getDictionaryFilePath("t1_c1").replace("\\", "/")
+ assertTrue(CarbonTablePath.getDictionaryFilePath(identifier.getTablePath(), "t1_c1").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.dict"));
- assertTrue(carbonTablePath.getDictionaryMetaFilePath("t1_c1").replace("\\", "/")
+ assertTrue(CarbonTablePath.getDictionaryMetaFilePath(identifier.getTablePath(), "t1_c1").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.dictmeta"));
- assertTrue(carbonTablePath.getSortIndexFilePath("t1_c1").replace("\\", "/")
+ assertTrue(CarbonTablePath.getSortIndexFilePath(identifier.getTablePath(),"t1_c1").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.sortindex"));
- assertTrue(carbonTablePath.getCarbonDataFilePath("2", 3, 4L, 0, 0, "999").replace("\\", "/")
+ assertTrue(CarbonTablePath.getCarbonDataFilePath(identifier.getTablePath(), "2", 3, 4L, 0, 0, "999").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Fact/Part0/Segment_2/part-3-4_batchno0-0-999.carbondata"));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatSharedDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatSharedDictionaryTest.java b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatSharedDictionaryTest.java
deleted file mode 100644
index 91384c1..0000000
--- a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatSharedDictionaryTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.util.path;
-
-import java.io.IOException;
-
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertTrue;
-
-/**
- * test shared dictionary paths
- */
-public class CarbonFormatSharedDictionaryTest {
-
- private final String CARBON_STORE = "/opt/carbonstore";
-
- /**
- * test shared dictionary location
- */
- @Test public void testSharedDimentionLocation() throws IOException {
- assertTrue(CarbonSharedDictionaryPath.getDictionaryFilePath(CARBON_STORE, "d1", "shared_c1").replace("\\", "/")
- .equals(CARBON_STORE + "/d1/SharedDictionary/shared_c1.dict"));
- assertTrue(CarbonSharedDictionaryPath.getDictionaryMetaFilePath(CARBON_STORE, "d1", "shared_c1").replace("\\", "/")
- .equals(CARBON_STORE + "/d1/SharedDictionary/shared_c1.dictmeta"));
- assertTrue(CarbonSharedDictionaryPath.getSortIndexFilePath(CARBON_STORE, "d1", "shared_c1").replace("\\", "/")
- .equals(CARBON_STORE + "/d1/SharedDictionary/shared_c1.sortindex"));
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
index 308d041..d5500e1 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
@@ -44,7 +44,6 @@ import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
import org.apache.carbondata.core.reader.CarbonDictionaryReaderImpl;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.ColumnDictionaryChunkMeta;
@@ -72,7 +71,7 @@ public class CarbonDictionaryWriterImplTest {
private String tableName;
- private String carbonStorePath;
+ private String tablePath;
private ColumnIdentifier columnIdentifier;
@@ -100,10 +99,10 @@ public class CarbonDictionaryWriterImplTest {
init();
this.databaseName = props.getProperty("database", "testSchema");
this.tableName = props.getProperty("tableName", "carbon");
- this.carbonStorePath = props.getProperty("storePath", "carbonStore");
+ this.tablePath = props.getProperty("storePath", "carbonStore");
this.columnIdentifier = new ColumnIdentifier("Name", null, null);
carbonTableIdentifier = new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
- absoluteTableIdentifier = AbsoluteTableIdentifier.from(carbonStorePath, carbonTableIdentifier);
+ absoluteTableIdentifier = AbsoluteTableIdentifier.from(tablePath, carbonTableIdentifier);
this.dictionaryColumnUniqueIdentifier =
new DictionaryColumnUniqueIdentifier(absoluteTableIdentifier, columnIdentifier,
columnIdentifier.getDataType());
@@ -479,8 +478,8 @@ public class CarbonDictionaryWriterImplTest {
* this method will delete the store path
*/
private void deleteStorePath() {
- FileFactory.FileType fileType = FileFactory.getFileType(this.carbonStorePath);
- CarbonFile carbonFile = FileFactory.getCarbonFile(this.carbonStorePath, fileType);
+ FileFactory.FileType fileType = FileFactory.getFileType(this.tablePath);
+ CarbonFile carbonFile = FileFactory.getCarbonFile(this.tablePath, fileType);
deleteRecursiveSilent(carbonFile);
}
@@ -528,14 +527,12 @@ public class CarbonDictionaryWriterImplTest {
* this method will form the dictionary directory paths
*/
private void initDictionaryDirPaths() throws IOException {
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(this.carbonStorePath, carbonTableIdentifier);
- String dictionaryLocation = carbonTablePath.getMetadataDirectoryPath();
+ String dictionaryLocation = CarbonTablePath.getMetadataPath(tablePath);
FileFactory.FileType fileType = FileFactory.getFileType(dictionaryLocation);
if(!FileFactory.isFileExist(dictionaryLocation, fileType)) {
FileFactory.mkdirs(dictionaryLocation, fileType);
}
- this.dictionaryFilePath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
- this.dictionaryMetaFilePath = carbonTablePath.getDictionaryMetaFilePath(columnIdentifier.getColumnId());
+ this.dictionaryFilePath = CarbonTablePath.getDictionaryFilePath(tablePath, columnIdentifier.getColumnId());
+ this.dictionaryMetaFilePath = CarbonTablePath.getDictionaryMetaFilePath(tablePath, columnIdentifier.getColumnId());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonBatchSparkStreamingExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonBatchSparkStreamingExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonBatchSparkStreamingExample.scala
index 6ae87b9..40460c6 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonBatchSparkStreamingExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonBatchSparkStreamingExample.scala
@@ -24,10 +24,6 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{CarbonEnv, SaveMode, SparkSession}
import org.apache.spark.streaming.{Seconds, StreamingContext, Time}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
-
/**
* This example introduces how to use CarbonData batch load to integrate
* with Spark Streaming(it's DStream, not Spark Structured Streaming)
@@ -74,7 +70,6 @@ object CarbonBatchSparkStreamingExample {
| """.stripMargin)
val carbonTable = CarbonEnv.getCarbonTable(Some("default"), streamTableName)(spark)
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
// batch load
val path = s"$rootPath/examples/spark2/src/main/resources/streamSample.csv"
spark.sql(
@@ -88,7 +83,7 @@ object CarbonBatchSparkStreamingExample {
val serverSocket = new ServerSocket(7071)
val thread1 = writeSocket(serverSocket)
val thread2 = showTableCount(spark, streamTableName)
- val ssc = startStreaming(spark, streamTableName, tablePath, checkpointPath)
+ val ssc = startStreaming(spark, streamTableName, checkpointPath)
// wait for stop signal to stop Spark Streaming App
waitForStopSignal(ssc)
// it need to start Spark Streaming App in main thread
@@ -153,7 +148,7 @@ object CarbonBatchSparkStreamingExample {
}
def startStreaming(spark: SparkSession, tableName: String,
- tablePath: CarbonTablePath, checkpointPath: String): StreamingContext = {
+ checkpointPath: String): StreamingContext = {
var ssc: StreamingContext = null
try {
// recommend: the batch interval must set larger, such as 30s, 1min.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStructuredStreamingExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStructuredStreamingExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStructuredStreamingExample.scala
index 247a59b..bca7fa3 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStructuredStreamingExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStructuredStreamingExample.scala
@@ -24,8 +24,9 @@ import org.apache.spark.sql.{CarbonEnv, SparkSession}
import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
// scalastyle:off println
object CarbonStructuredStreamingExample {
@@ -89,7 +90,6 @@ object CarbonStructuredStreamingExample {
}
val carbonTable = CarbonEnv.getCarbonTable(Some("default"), streamTableName)(spark)
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
// batch load
val path = s"$rootPath/examples/spark2/src/main/resources/streamSample.csv"
spark.sql(
@@ -101,7 +101,7 @@ object CarbonStructuredStreamingExample {
// streaming ingest
val serverSocket = new ServerSocket(7071)
- val thread1 = startStreaming(spark, tablePath)
+ val thread1 = startStreaming(spark, carbonTable)
val thread2 = writeSocket(serverSocket)
val thread3 = showTableCount(spark, streamTableName)
@@ -152,7 +152,7 @@ object CarbonStructuredStreamingExample {
thread
}
- def startStreaming(spark: SparkSession, tablePath: CarbonTablePath): Thread = {
+ def startStreaming(spark: SparkSession, carbonTable: CarbonTable): Thread = {
val thread = new Thread() {
override def run(): Unit = {
var qry: StreamingQuery = null
@@ -167,7 +167,8 @@ object CarbonStructuredStreamingExample {
qry = readSocketDF.writeStream
.format("carbondata")
.trigger(ProcessingTime("5 seconds"))
- .option("checkpointLocation", tablePath.getStreamingCheckpointDir)
+ .option("checkpointLocation",
+ CarbonTablePath.getStreamingCheckpointDir(carbonTable.getTablePath))
.option("dbName", "default")
.option("tableName", "stream_table")
.start()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index b1faa6a..24f5713 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -69,7 +69,6 @@ import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeConverter;
import org.apache.carbondata.core.util.DataTypeConverterImpl;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.BlockIndex;
import org.apache.carbondata.hadoop.CarbonInputSplit;
@@ -248,10 +247,6 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
}
}
- private static CarbonTablePath getTablePath(AbsoluteTableIdentifier absIdentifier) {
- return CarbonStorePath.getCarbonTablePath(absIdentifier);
- }
-
/**
* Set list of segments to access
*/
@@ -476,11 +471,10 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
List<InputSplit> splits = new ArrayList<InputSplit>();
if (streamSegments != null && !streamSegments.isEmpty()) {
- CarbonTablePath tablePath = CarbonStorePath.getCarbonTablePath(identifier);
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
for (String segmentId : streamSegments) {
- String segmentDir = tablePath.getSegmentDir(segmentId);
+ String segmentDir = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
if (FileFactory.isFileExist(segmentDir, fileType)) {
String indexName = CarbonTablePath.getCarbonStreamIndexFileName();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
index 3ef8afc..a4b3be8 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
@@ -37,7 +37,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.format.FileHeader;
import org.apache.carbondata.processing.loading.BadRecordsLogger;
@@ -127,9 +126,8 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
maxCacheSize = hadoopConf.getInt(CarbonStreamOutputFormat.CARBON_STREAM_CACHE_SIZE,
CarbonStreamOutputFormat.CARBON_STREAM_CACHE_SIZE_DEFAULT);
- CarbonTablePath tablePath =
- CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier());
- segmentDir = tablePath.getSegmentDir(segmentId);
+ segmentDir = CarbonTablePath.getSegmentPath(
+ carbonTable.getAbsoluteTableIdentifier().getTablePath(), segmentId);
fileName = CarbonTablePath.getCarbonDataFileName(0, taskNo, 0, 0, "0");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
index 423bb2a..dfa8dd1 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverte
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
/**
@@ -37,8 +37,7 @@ public class SchemaReader {
public static CarbonTable readCarbonTableFromStore(AbsoluteTableIdentifier identifier)
throws IOException {
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier);
- String schemaFilePath = carbonTablePath.getSchemaFilePath();
+ String schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath());
if (FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.LOCAL) ||
FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.HDFS) ||
FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.S3) ||
@@ -46,7 +45,7 @@ public class SchemaReader {
String tableName = identifier.getCarbonTableIdentifier().getTableName();
org.apache.carbondata.format.TableInfo tableInfo =
- CarbonUtil.readSchemaFile(carbonTablePath.getSchemaFilePath());
+ CarbonUtil.readSchemaFile(CarbonTablePath.getSchemaFilePath(identifier.getTablePath()));
SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
tableInfo,
@@ -63,22 +62,21 @@ public class SchemaReader {
/**
* the method returns the Wrapper TableInfo
*
- * @param absoluteTableIdentifier
+ * @param identifier
* @return
*/
- public static TableInfo getTableInfo(AbsoluteTableIdentifier absoluteTableIdentifier)
+ public static TableInfo getTableInfo(AbsoluteTableIdentifier identifier)
throws IOException {
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
org.apache.carbondata.format.TableInfo thriftTableInfo =
- CarbonUtil.readSchemaFile(carbonTablePath.getSchemaFilePath());
+ CarbonUtil.readSchemaFile(CarbonTablePath.getSchemaFilePath(identifier.getTablePath()));
ThriftWrapperSchemaConverterImpl thriftWrapperSchemaConverter =
new ThriftWrapperSchemaConverterImpl();
CarbonTableIdentifier carbonTableIdentifier =
- absoluteTableIdentifier.getCarbonTableIdentifier();
+ identifier.getCarbonTableIdentifier();
return thriftWrapperSchemaConverter.fromExternalToWrapperTableInfo(
thriftTableInfo,
carbonTableIdentifier.getDatabaseName(),
carbonTableIdentifier.getTableName(),
- absoluteTableIdentifier.getTablePath());
+ identifier.getTablePath());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index ac17c4e..6365a6a 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -65,7 +65,6 @@ import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
@@ -188,11 +187,11 @@ public class StoreCreator {
}
public static CarbonTable createTable(
- AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
+ AbsoluteTableIdentifier identifier) throws IOException {
TableInfo tableInfo = new TableInfo();
- tableInfo.setDatabaseName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName());
+ tableInfo.setDatabaseName(identifier.getCarbonTableIdentifier().getDatabaseName());
TableSchema tableSchema = new TableSchema();
- tableSchema.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
+ tableSchema.setTableName(identifier.getCarbonTableIdentifier().getTableName());
List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
ArrayList<Encoding> encodings = new ArrayList<>();
encodings.add(Encoding.DICTIONARY);
@@ -284,13 +283,12 @@ public class StoreCreator {
tableSchema.setSchemaEvalution(schemaEvol);
tableSchema.setTableId(UUID.randomUUID().toString());
tableInfo.setTableUniqueName(
- absoluteTableIdentifier.getCarbonTableIdentifier().getTableUniqueName()
+ identifier.getCarbonTableIdentifier().getTableUniqueName()
);
tableInfo.setLastUpdatedTime(System.currentTimeMillis());
tableInfo.setFactTable(tableSchema);
- tableInfo.setTablePath(absoluteTableIdentifier.getTablePath());
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- String schemaFilePath = carbonTablePath.getSchemaFilePath();
+ tableInfo.setTablePath(identifier.getTablePath());
+ String schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath());
String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath);
CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
@@ -469,7 +467,7 @@ public class StoreCreator {
loadMetadataDetails.setLoadStartTime(loadMetadataDetails.getTimeStamp(readCurrentTime()));
listOfLoadFolderDetails.add(loadMetadataDetails);
- String dataLoadLocation = schema.getCarbonTable().getMetaDataFilepath() + File.separator
+ String dataLoadLocation = schema.getCarbonTable().getMetadataPath() + File.separator
+ CarbonCommonConstants.LOADMETADATA_FILENAME;
DataOutputStream dataOutputStream;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
index 5a2f831..8f7e88c 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
@@ -25,7 +25,6 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.service.impl.PathFactory;
import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.CarbonProjection;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
@@ -82,7 +81,8 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider {
carbonTableReader.getCarbonCache(carbondataSplit.getSchemaTableName());
checkNotNull(tableCacheModel, "tableCacheModel should not be null");
checkNotNull(tableCacheModel.carbonTable, "tableCacheModel.carbonTable should not be null");
- checkNotNull(tableCacheModel.tableInfo, "tableCacheModel.tableInfo should not be null");
+ checkNotNull(
+ tableCacheModel.carbonTable.getTableInfo(), "tableCacheModel.tableInfo should not be null");
// Build Query Model
CarbonTable targetTable = tableCacheModel.carbonTable;
@@ -92,8 +92,7 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider {
try {
Configuration conf = new Configuration();
conf.set(CarbonTableInputFormat.INPUT_SEGMENT_NUMBERS, "");
- String carbonTablePath = PathFactory.getInstance()
- .getCarbonTablePath(targetTable.getAbsoluteTableIdentifier(), null).getPath();
+ String carbonTablePath = targetTable.getAbsoluteTableIdentifier().getTablePath();
conf.set(CarbonTableInputFormat.INPUT_DIR, carbonTablePath);
JobConf jobConf = new JobConf(conf);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
index 8422c3e..4984406 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
@@ -17,27 +17,18 @@
package org.apache.carbondata.presto.impl;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
/**
- * Caching metadata of CarbonData(e.g. TableIdentifier, TablePath, TableInfo, CarbonTable) in Class CarbonTableReader
+ * Caching metadata of CarbonData in Class CarbonTableReader
* to speed up query
*/
public class CarbonTableCacheModel {
- public CarbonTableIdentifier carbonTableIdentifier;
- public CarbonTablePath carbonTablePath;
-
- public TableInfo tableInfo;
public CarbonTable carbonTable;
public boolean isValid() {
- if (carbonTable != null && carbonTablePath != null && carbonTableIdentifier != null)
- return true;
- else return false;
+ return carbonTable != null;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index b0271ef..09389f8 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -42,7 +42,6 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.service.impl.PathFactory;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
@@ -243,9 +242,10 @@ public class CarbonTableReader {
updateSchemaList();
}
try {
- if (isKeyExists && !FileFactory
- .isFileExist(carbonCache.get().get(schemaTableName).carbonTablePath.getSchemaFilePath(),
- fileType)) {
+ if (isKeyExists
+ && !FileFactory.isFileExist(
+ CarbonTablePath.getSchemaFilePath(
+ carbonCache.get().get(schemaTableName).carbonTable.getTablePath()), fileType)) {
removeTableFromCache(schemaTableName);
throw new TableNotFoundException(schemaTableName);
}
@@ -255,10 +255,12 @@ public class CarbonTableReader {
if (isKeyExists) {
CarbonTableCacheModel ctcm = carbonCache.get().get(schemaTableName);
- if(ctcm != null && ctcm.tableInfo != null) {
- Long latestTime = FileFactory.getCarbonFile(ctcm.carbonTablePath.getSchemaFilePath())
- .getLastModifiedTime();
- Long oldTime = ctcm.tableInfo.getLastUpdatedTime();
+ if(ctcm != null && ctcm.carbonTable.getTableInfo() != null) {
+ Long latestTime = FileFactory.getCarbonFile(
+ CarbonTablePath.getSchemaFilePath(
+ carbonCache.get().get(schemaTableName).carbonTable.getTablePath())
+ ).getLastModifiedTime();
+ Long oldTime = ctcm.carbonTable.getTableInfo().getLastUpdatedTime();
if (DateUtils.truncate(new Date(latestTime), Calendar.MINUTE)
.after(DateUtils.truncate(new Date(oldTime), Calendar.MINUTE))) {
removeTableFromCache(schemaTableName);
@@ -312,21 +314,12 @@ public class CarbonTableReader {
// Step 1: get store path of the table and cache it.
// create table identifier. the table id is randomly generated.
- cache.carbonTableIdentifier =
+ CarbonTableIdentifier carbonTableIdentifier =
new CarbonTableIdentifier(table.getSchemaName(), table.getTableName(),
UUID.randomUUID().toString());
String storePath = config.getStorePath();
- String tablePath = storePath + "/" + cache.carbonTableIdentifier.getDatabaseName() + "/"
- + cache.carbonTableIdentifier.getTableName();
-
- // get the store path of the table.
-
- AbsoluteTableIdentifier absoluteTableIdentifier =
- AbsoluteTableIdentifier.from(tablePath, cache.carbonTableIdentifier);
- cache.carbonTablePath =
- PathFactory.getInstance().getCarbonTablePath(absoluteTableIdentifier, null);
- // cache the table
- carbonCache.get().put(table, cache);
+ String tablePath = storePath + "/" + carbonTableIdentifier.getDatabaseName() + "/"
+ + carbonTableIdentifier.getTableName();
//Step 2: read the metadata (tableInfo) of the table.
ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
@@ -338,7 +331,7 @@ public class CarbonTableReader {
}
};
ThriftReader thriftReader =
- new ThriftReader(cache.carbonTablePath.getSchemaFilePath(), createTBase);
+ new ThriftReader(CarbonTablePath.getSchemaFilePath(tablePath), createTBase);
thriftReader.open();
org.apache.carbondata.format.TableInfo tableInfo =
(org.apache.carbondata.format.TableInfo) thriftReader.read();
@@ -355,9 +348,12 @@ public class CarbonTableReader {
// Step 4: Load metadata info into CarbonMetadata
CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
- cache.tableInfo = wrapperTableInfo;
- cache.carbonTable = CarbonMetadata.getInstance()
- .getCarbonTable(cache.carbonTableIdentifier.getTableUniqueName());
+ cache.carbonTable = CarbonMetadata.getInstance().getCarbonTable(
+ table.getSchemaName(), table.getTableName());
+
+ // cache the table
+ carbonCache.get().put(table, cache);
+
result = cache.carbonTable;
} catch (Exception ex) {
throw new RuntimeException(ex);
@@ -372,11 +368,10 @@ public class CarbonTableReader {
List<CarbonLocalInputSplit> result = new ArrayList<>();
CarbonTable carbonTable = tableCacheModel.carbonTable;
- TableInfo tableInfo = tableCacheModel.tableInfo;
+ TableInfo tableInfo = tableCacheModel.carbonTable.getTableInfo();
Configuration config = new Configuration();
config.set(CarbonTableInputFormat.INPUT_SEGMENT_NUMBERS, "");
- String carbonTablePath = PathFactory.getInstance()
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier(), null).getPath();
+ String carbonTablePath = carbonTable.getAbsoluteTableIdentifier().getTablePath();
config.set(CarbonTableInputFormat.INPUT_DIR, carbonTablePath);
config.set(CarbonTableInputFormat.DATABASE_NAME, carbonTable.getDatabaseName());
config.set(CarbonTableInputFormat.TABLE_NAME, carbonTable.getTableName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index a41e738..1d7c791 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -52,7 +52,7 @@ import org.apache.carbondata.core.metadata.schema.{SchemaEvolution, SchemaEvolut
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata,
CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.core.writer.sortindex.{CarbonDictionarySortIndexWriter,
CarbonDictionarySortIndexWriterImpl, CarbonDictionarySortInfo, CarbonDictionarySortInfoPreparator}
@@ -323,10 +323,8 @@ object CarbonDataStoreCreator {
)
tableInfo.setLastUpdatedTime(System.currentTimeMillis())
tableInfo.setFactTable(tableSchema)
- val carbonTablePath: CarbonTablePath = CarbonStorePath.getCarbonTablePath(
- absoluteTableIdentifier.getTablePath,
- absoluteTableIdentifier.getCarbonTableIdentifier)
- val schemaFilePath: String = carbonTablePath.getSchemaFilePath
+ val schemaFilePath: String = CarbonTablePath.getSchemaFilePath(
+ absoluteTableIdentifier.getTablePath)
val schemaMetadataPath: String =
CarbonTablePath.getFolderContainingFile(schemaFilePath)
CarbonMetadata.getInstance.loadTableMetadata(tableInfo)
@@ -533,7 +531,7 @@ object CarbonDataStoreCreator {
loadMetadataDetails.setLoadStartTime(
loadMetadataDetails.getTimeStamp(readCurrentTime()))
listOfLoadFolderDetails.add(loadMetadataDetails)
- val dataLoadLocation: String = schema.getCarbonTable.getMetaDataFilepath + File.separator +
+ val dataLoadLocation: String = schema.getCarbonTable.getMetadataPath + File.separator +
CarbonCommonConstants.LOADMETADATA_FILENAME
val gsonObjectToWrite: Gson = new Gson()
val writeOperation: AtomicFileOperations = new AtomicFileOperationsImpl(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
index cb0d02c..5eb274d 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
@@ -28,7 +28,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
/**
* Test Class for AlterTableTestCase to verify all scenerios
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index c84ae6b..a707c60 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -24,7 +24,7 @@ import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.Row
import org.scalatest.BeforeAndAfterEach
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.spark.sql.test.util.QueryTest
@@ -48,13 +48,13 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
datbaseName: String,
tableName: String): Boolean = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(datbaseName, tableName)
- val partitionPath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir
+ val partitionPath =
+ CarbonTablePath.getPartitionDir(carbonTable.getAbsoluteTableIdentifier.getTablePath)
val fileType: FileFactory.FileType = FileFactory.getFileType(partitionPath)
val carbonFile = FileFactory.getCarbonFile(partitionPath, fileType)
val segments: ArrayBuffer[String] = ArrayBuffer()
carbonFile.listFiles.foreach { file =>
- segments += CarbonTablePath.DataPathUtil.getSegmentId(file.getAbsolutePath + "/dummy")
+ segments += CarbonTablePath.DataFileUtil.getSegmentId(file.getAbsolutePath + "/dummy")
}
segments.contains(segmentId)
}
@@ -235,8 +235,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
sql("create table stale(a string) stored by 'carbondata'")
sql("insert into stale values('k')")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "stale")
- val tableStatusFile = new CarbonTablePath(null,
- carbonTable.getTablePath).getTableStatusFilePath
+ val tableStatusFile = CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath)
FileFactory.getCarbonFile(tableStatusFile).delete()
sql("insert into stale values('k')")
checkAnswer(sql("select * from stale"), Row("k"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index 5cc4156..3c2fd71 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -29,7 +29,7 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
var timeStampPropOrig: String = _
@@ -231,8 +231,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("insert overwrite table HiveOverwrite select * from THive")
checkAnswer(sql("select count(*) from CarbonOverwrite"), sql("select count(*) from HiveOverwrite"))
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbonoverwrite")
- val partitionPath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir
+ val partitionPath = CarbonTablePath.getPartitionDir(carbonTable.getAbsoluteTableIdentifier.getTablePath)
val folder = new File(partitionPath)
assert(folder.isDirectory)
assert(folder.list().length == 1)
@@ -254,8 +253,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA local INPATH '$resourcesPath/100_olap.csv' overwrite INTO TABLE HiveOverwrite")
checkAnswer(sql("select count(*) from TCarbonSourceOverwrite"), sql("select count(*) from HiveOverwrite"))
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "tcarbonsourceoverwrite")
- val partitionPath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir
+ val partitionPath = CarbonTablePath.getPartitionDir(carbonTable.getAbsoluteTableIdentifier.getTablePath)
val folder = new File(partitionPath)
assert(folder.isDirectory)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/concurrent/TestLoadTableConcurrentScenario.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/concurrent/TestLoadTableConcurrentScenario.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/concurrent/TestLoadTableConcurrentScenario.scala
deleted file mode 100644
index 6af28c3..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/concurrent/TestLoadTableConcurrentScenario.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.concurrent
-
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
-import org.apache.spark.sql.CarbonEnv
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-class TestLoadTableConcurrentScenario extends QueryTest with BeforeAndAfterAll {
-
- var carbonTable: CarbonTable = _
- var metaPath: String = _
-
- override def beforeAll {
- sql("use default")
- sql("drop table if exists drop_concur")
- sql("drop table if exists rename_concur")
- }
-
- test("do not allow drop table when load is in progress") {
- sql("create table drop_concur(id int, name string) stored by 'carbondata'")
- sql("insert into drop_concur select 1,'abc'")
- sql("insert into drop_concur select 1,'abc'")
- sql("insert into drop_concur select 1,'abc'")
-
- carbonTable = CarbonEnv.getCarbonTable(Option("default"), "drop_concur")(sqlContext.sparkSession)
- metaPath = carbonTable.getMetaDataFilepath
- val listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath)
- listOfLoadFolderDetailsArray(1).setSegmentStatus(SegmentStatus.INSERT_IN_PROGRESS)
-
- try {
- sql("drop table drop_concur")
- } catch {
- case ex: Throwable => assert(ex.getMessage.contains("Cannot drop table, load or insert overwrite is in progress"))
- }
- }
-
- test("do not allow rename table when load is in progress") {
- sql("create table rename_concur(id int, name string) stored by 'carbondata'")
- sql("insert into rename_concur select 1,'abc'")
- sql("insert into rename_concur select 1,'abc'")
-
- carbonTable = CarbonEnv.getCarbonTable(Option("default"), "rename_concur")(sqlContext.sparkSession)
- metaPath = carbonTable.getMetaDataFilepath
- val listOfLoadFolderDetailsArray = SegmentStatusManager.readLoadMetadata(metaPath)
- listOfLoadFolderDetailsArray(1).setSegmentStatus(SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)
-
- try {
- sql("alter table rename_concur rename to rename_concur1")
- } catch {
- case ex: Throwable => assert(ex.getMessage.contains("alter rename failed, load, insert or insert overwrite " +
- "is in progress for the table"))
- }
- }
-
- override def afterAll: Unit = {
- sql("use default")
- sql("drop table if exists drop_concur")
- sql("drop table if exists rename_concur")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
index 09268b5..8315848 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
@@ -142,7 +142,7 @@ class TestCreateTableAsSelect extends QueryTest with BeforeAndAfterAll {
val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetastore
.lookupRelation(Option("default"), "ctas_tblproperties_test")(Spark2TestQueryExecutor.spark)
.asInstanceOf[CarbonRelation].carbonTable
- val metadataFolderPath: CarbonFile = FileFactory.getCarbonFile(carbonTable.getMetaDataFilepath)
+ val metadataFolderPath: CarbonFile = FileFactory.getCarbonFile(carbonTable.getMetadataPath)
assert(metadataFolderPath.exists())
val dictFiles: Array[CarbonFile] = metadataFolderPath.listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
index 2ef88a4..a7607c3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
@@ -24,7 +24,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.spark.sql.test.util.QueryTest
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
@@ -43,9 +43,7 @@ class DataCompactionLockTest extends QueryTest with BeforeAndAfterAll {
new CarbonTableIdentifier(
CarbonCommonConstants.DATABASE_DEFAULT_NAME, "compactionlocktesttable", "1")
)
- val carbonTablePath: CarbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier)
- val dataPath: String = carbonTablePath.getMetadataDirectoryPath
+ val dataPath: String = CarbonTablePath.getMetadataPath(absoluteTableIdentifier.getTablePath)
val carbonLock: ICarbonLock =
CarbonLockFactory
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index 86f974c..5310148 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.hadoop.CacheClient
import org.apache.spark.sql.test.util.QueryTest
@@ -112,11 +112,9 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
CarbonCommonConstants.DATABASE_DEFAULT_NAME,
"ignoremajor"
)
- val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
- val carbontablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)
- .getMetadataDirectoryPath
- val segs = SegmentStatusManager.readLoadMetadata(carbontablePath)
+ val carbonTablePath = carbonTable.getMetadataPath
+ val segs = SegmentStatusManager.readLoadMetadata(carbonTablePath)
// status should remain as compacted.
assertResult(SegmentStatus.COMPACTED)(segs(3).getSegmentStatus)
@@ -134,9 +132,7 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
CarbonCommonConstants.DATABASE_DEFAULT_NAME,
"ignoremajor"
)
- val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
- val carbontablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier).getMetadataDirectoryPath
+ val carbontablePath = carbonTable.getMetadataPath
val segs = SegmentStatusManager.readLoadMetadata(carbontablePath)
// status should remain as compacted for segment 2.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
index 42ac4df..68a3058 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.test.util.QueryTest
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
var filePath: String = _
@@ -193,8 +193,7 @@ class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
CarbonCommonConstants.DATABASE_DEFAULT_NAME,
tableName
)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getCarbonDataDirectoryPath(segmentNo)
+ val segmentDir = carbonTable.getSemgentPath(segmentNo)
new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
index db0a62c..b9d8e12 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
@@ -22,7 +22,7 @@ import java.io.{File, FilenameFilter}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.reader.CarbonIndexFileReader
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
@@ -48,8 +48,7 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$testData' into table test_table_v3")
val indexReader = new CarbonIndexFileReader()
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "test_table_v3")
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getCarbonDataDirectoryPath("0")
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, "0")
val carbonIndexPaths = new File(segmentDir)
.listFiles(new FilenameFilter {
override def accept(dir: File, name: String): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index fe5e327..5e5eed5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -32,8 +32,8 @@ import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.spark.rdd.CarbonScanRDD
+import org.apache.carbondata.core.util.path.CarbonTablePath
class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
var filePath: String = s"$resourcesPath/globalsort"
@@ -271,8 +271,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
}
sql(s"LOAD DATA LOCAL INPATH '$inputPath' INTO TABLE carbon_globalsort")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbon_globalsort")
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getSegmentDir("0")
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, "0")
assertResult(Math.max(4, defaultParallelism) + 1)(new File(segmentDir).listFiles().length)
}
@@ -378,8 +377,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
private def getIndexFileCount(tableName: String, segmentNo: String = "0"): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", tableName)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getCarbonDataDirectoryPath(segmentNo)
+ val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentNo)
new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
index ed58253..7c82f75 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
@@ -25,7 +25,7 @@ import org.apache.commons.lang3.time.DateUtils
import org.apache.spark.sql.Row
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
@@ -79,8 +79,8 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
"dataRetentionTable"
)
absoluteTableIdentifierForRetention = carbonTable2.getAbsoluteTableIdentifier
- carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifierForRetention).getMetadataDirectoryPath
+ carbonTablePath = CarbonTablePath
+ .getMetadataPath(absoluteTableIdentifierForRetention.getTablePath)
carbonTableStatusLock = CarbonLockFactory
.getCarbonLockObj(absoluteTableIdentifierForLock, LockUsage.TABLE_STATUS_LOCK)
carbonDeleteSegmentLock= CarbonLockFactory
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
index 0a21aed..e5de8da 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
@@ -61,9 +61,7 @@ class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll
def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Seq[Int]): Unit = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
+ val segmentDir = carbonTable.getSemgentPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
index 5427981..877d026 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
@@ -51,9 +51,7 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int, partitionMapFiles: Int): Unit = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
+ val segmentDir = carbonTable.getSemgentPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
index f4b6e0e..40da6f1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
@@ -51,9 +51,7 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int): Unit = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
+ val segmentDir = carbonTable.getSemgentPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 7fcfb29..6358c83 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -66,9 +66,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int): Unit = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
+ val segmentDir = carbonTable.getSemgentPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
@@ -333,9 +331,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE mergeindexpartitionthree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_mergeindexpartitionthree")
- val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0")
+ val segmentDir = carbonTable.getSemgentPath("0")
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val files = carbonFile.listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = CarbonTablePath.isCarbonIndexFile(file.getName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index c02ba0a..b98bddf 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -173,7 +173,7 @@ object CarbonStore {
LOGGER.audit(s"Delete segment by Id request has been received for $dbName.$tableName")
validateLoadIds(loadids)
- val path = carbonTable.getMetaDataFilepath
+ val path = carbonTable.getMetadataPath
try {
val invalidLoadIds = SegmentStatusManager.updateDeletionStatus(
@@ -199,7 +199,7 @@ object CarbonStore {
LOGGER.audit(s"Delete segment by Id request has been received for $dbName.$tableName")
val time = validateTimeFormat(timestamp)
- val path = carbonTable.getMetaDataFilepath
+ val path = carbonTable.getMetadataPath
try {
val invalidLoadTimestamps =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
index 32d121e..3dd9903 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
@@ -30,7 +30,7 @@ trait ColumnValidator {
*/
trait DictionaryDetailService {
def getDictionaryDetail(dictFolderPath: String, primDimensions: Array[CarbonDimension],
- table: CarbonTableIdentifier, storePath: String): DictionaryDetail
+ tablePath: String): DictionaryDetail
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
index e861a8c..dbf47ab 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
@@ -23,12 +23,11 @@ import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFi
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
class DictionaryDetailHelper extends DictionaryDetailService {
- def getDictionaryDetail(dictfolderPath: String, primDimensions: Array[CarbonDimension],
- table: CarbonTableIdentifier, storePath: String): DictionaryDetail = {
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, table)
+ override def getDictionaryDetail(dictfolderPath: String, primDimensions: Array[CarbonDimension],
+ tablePath: String): DictionaryDetail = {
val dictFilePaths = new Array[String](primDimensions.length)
val dictFileExists = new Array[Boolean](primDimensions.length)
val columnIdentifier = new Array[ColumnIdentifier](primDimensions.length)
@@ -50,7 +49,7 @@ class DictionaryDetailHelper extends DictionaryDetailService {
// 3 lookup fileNamesMap, if file name is in fileNamesMap, file is exists, or not.
primDimensions.zipWithIndex.foreach { f =>
columnIdentifier(f._2) = f._1.getColumnIdentifier
- dictFilePaths(f._2) = carbonTablePath.getDictionaryFilePath(f._1.getColumnId)
+ dictFilePaths(f._2) = CarbonTablePath.getDictionaryFilePath(tablePath, f._1.getColumnId)
dictFileExists(f._2) =
fileNamesMap.get(CarbonTablePath.getDictionaryFileName(f._1.getColumnId)) match {
case None => false
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
index f2f4ecd..56a66b9 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.statusmanager.SegmentStatus
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.spark.util.GlobalDictionaryUtil
/**
@@ -49,7 +49,7 @@ class AddColumnPartition(rddId: Int, idx: Int, schema: ColumnSchema) extends Par
*/
class AlterTableAddColumnRDD[K, V](sc: SparkContext,
@transient newColumns: Seq[ColumnSchema],
- absoluteTableIdentifier: AbsoluteTableIdentifier)
+ identifier: AbsoluteTableIdentifier)
extends CarbonRDD[(Int, SegmentStatus)](sc, Nil) {
val lockType: String = CarbonProperties.getInstance.getProperty(CarbonCommonConstants.LOCK_TYPE,
@@ -70,8 +70,6 @@ class AlterTableAddColumnRDD[K, V](sc: SparkContext,
// create dictionary file if it is a dictionary column
if (columnSchema.hasEncoding(Encoding.DICTIONARY) &&
!columnSchema.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier)
var rawData: String = null
if (null != columnSchema.getDefaultValue) {
rawData = new String(columnSchema.getDefaultValue,
@@ -79,16 +77,15 @@ class AlterTableAddColumnRDD[K, V](sc: SparkContext,
}
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.LOCK_TYPE, lockType)
// Create table and metadata folders if not exist
- val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
+ val metadataDirectoryPath = CarbonTablePath.getMetadataPath(identifier.getTablePath)
val fileType = FileFactory.getFileType(metadataDirectoryPath)
if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
FileFactory.mkdirs(metadataDirectoryPath, fileType)
}
- GlobalDictionaryUtil
- .loadDefaultDictionaryValueForNewColumn(carbonTablePath,
- columnSchema,
- absoluteTableIdentifier,
- rawData)
+ GlobalDictionaryUtil.loadDefaultDictionaryValueForNewColumn(
+ columnSchema,
+ identifier,
+ rawData)
}
} catch {
case ex: Exception =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3e99681/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index db29532..7acf4e2 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -40,10 +40,8 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.service.{CarbonCommonFactory, PathService}
import org.apache.carbondata.core.statusmanager.SegmentStatus
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.processing.loading.exception.NoRetryException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.util.CarbonLoaderUtil
@@ -348,10 +346,6 @@ class CarbonGlobalDictionaryGenerateRDD(
model.table,
model.columnIdentifier(split.index),
model.columnIdentifier(split.index).getDataType)
- val pathService: PathService = CarbonCommonFactory.getPathService
- val carbonTablePath: CarbonTablePath =
- pathService
- .getCarbonTablePath(model.table, dictionaryColumnUniqueIdentifier)
if (StringUtils.isNotBlank(model.hdfsTempLocation)) {
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION,
model.hdfsTempLocation)
[50/50] [abbrv] carbondata git commit: [CARBONDATA-1544][Datamap]
Datamap FineGrain implementation
Posted by ra...@apache.org.
[CARBONDATA-1544][Datamap] Datamap FineGrain implementation
Implemented interfaces for FG datamap and integrated to filterscanner to use the pruned bitset from FG datamap.
FG Query flow as follows.
1.The user can add FG datamap to any table and implement there interfaces.
2. Any filter query which hits the table with datamap will call prune method of FGdatamap.
3. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the information of block, blocklet, page and rowids information as well.
4. The pruned blocklets are internally wriitten to file and returns only the block , blocklet and filepath information as part of Splits.
5. Based on the splits scanrdd schedule the tasks.
6. In filterscanner we check the datamapwriterpath from split and reNoteads the bitset if exists. And pass this bitset as input to it.
This closes #1471
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e972fd3d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e972fd3d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e972fd3d
Branch: refs/heads/fgdatamap
Commit: e972fd3d5cc8f392d47ca111b2d8f262edb29ac6
Parents: cae74a8
Author: ravipesala <ra...@gmail.com>
Authored: Wed Nov 15 19:48:40 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Feb 5 19:28:52 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/datamap/DataMapMeta.java | 8 +-
.../core/datamap/DataMapStoreManager.java | 30 +-
.../carbondata/core/datamap/DataMapType.java | 21 +
.../carbondata/core/datamap/TableDataMap.java | 30 +-
.../core/datamap/dev/AbstractDataMapWriter.java | 110 +++++
.../core/datamap/dev/BlockletSerializer.java | 57 +++
.../carbondata/core/datamap/dev/DataMap.java | 4 +-
.../core/datamap/dev/DataMapFactory.java | 14 +-
.../core/datamap/dev/DataMapWriter.java | 57 ---
.../cgdatamap/AbstractCoarseGrainDataMap.java | 24 +
.../AbstractCoarseGrainDataMapFactory.java | 34 ++
.../dev/fgdatamap/AbstractFineGrainDataMap.java | 24 +
.../AbstractFineGrainDataMapFactory.java | 38 ++
.../carbondata/core/datastore/DataRefNode.java | 6 +
.../core/datastore/block/TableBlockInfo.java | 10 +
.../impl/btree/AbstractBTreeLeafNode.java | 5 +
.../datastore/impl/btree/BTreeNonLeafNode.java | 5 +
.../carbondata/core/indexstore/Blocklet.java | 30 +-
.../indexstore/BlockletDataMapIndexStore.java | 6 -
.../core/indexstore/BlockletDetailsFetcher.java | 8 +
.../core/indexstore/ExtendedBlocklet.java | 17 +
.../core/indexstore/FineGrainBlocklet.java | 120 +++++
.../blockletindex/BlockletDataMap.java | 15 +-
.../blockletindex/BlockletDataMapFactory.java | 63 ++-
.../blockletindex/BlockletDataRefNode.java | 27 +-
.../indexstore/blockletindex/IndexWrapper.java | 18 +
.../core/indexstore/schema/FilterType.java | 24 -
.../executer/ExcludeFilterExecuterImpl.java | 3 +
.../executer/IncludeFilterExecuterImpl.java | 3 +
.../scanner/impl/BlockletFilterScanner.java | 2 +
.../apache/carbondata/core/util/CarbonUtil.java | 97 ++++
.../datamap/examples/MinMaxDataMap.java | 32 +-
.../datamap/examples/MinMaxDataMapFactory.java | 49 ++-
.../datamap/examples/MinMaxDataWriter.java | 36 +-
.../examples/MinMaxIndexBlockDetails.java | 13 -
.../carbondata/hadoop/CarbonInputSplit.java | 21 +-
.../hadoop/api/CarbonTableInputFormat.java | 17 +-
.../testsuite/datamap/CGDataMapTestCase.scala | 361 +++++++++++++++
.../testsuite/datamap/DataMapWriterSuite.scala | 46 +-
.../testsuite/datamap/FGDataMapTestCase.scala | 440 +++++++++++++++++++
.../iud/InsertOverwriteConcurrentTest.scala | 22 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 7 +-
.../TestStreamingTableOperation.scala | 5 +-
.../datamap/DataMapWriterListener.java | 57 ++-
.../store/CarbonFactDataHandlerModel.java | 10 +-
.../store/writer/AbstractFactDataWriter.java | 128 +-----
46 files changed, 1763 insertions(+), 391 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
index 7746acf..dd15ccb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
@@ -19,15 +19,15 @@ package org.apache.carbondata.core.datamap;
import java.util.List;
-import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
public class DataMapMeta {
private List<String> indexedColumns;
- private FilterType optimizedOperation;
+ private List<ExpressionType> optimizedOperation;
- public DataMapMeta(List<String> indexedColumns, FilterType optimizedOperation) {
+ public DataMapMeta(List<String> indexedColumns, List<ExpressionType> optimizedOperation) {
this.indexedColumns = indexedColumns;
this.optimizedOperation = optimizedOperation;
}
@@ -36,7 +36,7 @@ public class DataMapMeta {
return indexedColumns;
}
- public FilterType getOptimizedOperation() {
+ public List<ExpressionType> getOptimizedOperation() {
return optimizedOperation;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 90e5fff..8d80b4d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -56,7 +56,22 @@ public final class DataMapStoreManager {
}
public List<TableDataMap> getAllDataMap(AbsoluteTableIdentifier identifier) {
- return allDataMaps.get(identifier.uniqueName());
+ return allDataMaps.get(identifier.getCarbonTableIdentifier().getTableUniqueName());
+ }
+
+ // TODO its a temporary method till chooser is implemented
+ public TableDataMap chooseDataMap(AbsoluteTableIdentifier identifier) {
+ List<TableDataMap> tableDataMaps = getAllDataMap(identifier);
+ if (tableDataMaps != null && tableDataMaps.size() > 0) {
+ for (TableDataMap dataMap: tableDataMaps) {
+ if (!dataMap.getDataMapName().equalsIgnoreCase(BlockletDataMap.NAME)) {
+ return dataMap;
+ }
+ }
+ return tableDataMaps.get(0);
+ } else {
+ return getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName());
+ }
}
/**
@@ -68,7 +83,7 @@ public final class DataMapStoreManager {
*/
public TableDataMap getDataMap(AbsoluteTableIdentifier identifier,
String dataMapName, String factoryClass) {
- String table = identifier.uniqueName();
+ String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
List<TableDataMap> tableDataMaps = allDataMaps.get(table);
TableDataMap dataMap;
if (tableDataMaps == null) {
@@ -96,7 +111,7 @@ public final class DataMapStoreManager {
*/
public TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
String factoryClassName, String dataMapName) {
- String table = identifier.uniqueName();
+ String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
// Just update the segmentRefreshMap with the table if not added.
getTableSegmentRefresher(identifier);
List<TableDataMap> tableDataMaps = allDataMaps.get(table);
@@ -149,7 +164,9 @@ public final class DataMapStoreManager {
* @param identifier Table identifier
*/
public void clearDataMaps(AbsoluteTableIdentifier identifier) {
- List<TableDataMap> tableDataMaps = allDataMaps.get(identifier.uniqueName());
+ String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName();
+ List<TableDataMap> tableDataMaps =
+ allDataMaps.get(tableUniqueName);
segmentRefreshMap.remove(identifier.uniqueName());
if (tableDataMaps != null) {
for (TableDataMap tableDataMap: tableDataMaps) {
@@ -158,7 +175,7 @@ public final class DataMapStoreManager {
break;
}
}
- allDataMaps.remove(identifier.uniqueName());
+ allDataMaps.remove(tableUniqueName);
}
}
@@ -167,7 +184,8 @@ public final class DataMapStoreManager {
* @param identifier Table identifier
*/
public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) {
- List<TableDataMap> tableDataMaps = allDataMaps.get(identifier.uniqueName());
+ List<TableDataMap> tableDataMaps =
+ allDataMaps.get(identifier.getCarbonTableIdentifier().getTableUniqueName());
if (tableDataMaps != null) {
int i = 0;
for (TableDataMap tableDataMap: tableDataMaps) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
new file mode 100644
index 0000000..bf812b3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap;
+
+public enum DataMapType {
+ CG,FG;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 1c80703..42fc702 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -21,12 +21,15 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -76,10 +79,15 @@ public final class TableDataMap extends OperationEventListener {
SegmentProperties segmentProperties;
for (String segmentId : segmentIds) {
List<Blocklet> pruneBlocklets = new ArrayList<>();
- List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
- segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId);
- for (DataMap dataMap : dataMaps) {
- pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions));
+ // if filter is not passed then return all the blocklets
+ if (filterExp == null) {
+ pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segmentId, partitions);
+ } else {
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
+ segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segmentId);
+ for (DataMap dataMap : dataMaps) {
+ pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions));
+ }
}
blocklets.addAll(addSegmentId(blockletDetailsFetcher
.getExtendedBlocklets(pruneBlocklets, segmentId), segmentId));
@@ -137,9 +145,21 @@ public final class TableDataMap extends OperationEventListener {
segmentPropertiesFetcher.getSegmentProperties(distributable.getSegmentId()),
partitions));
}
- for (Blocklet blocklet: blocklets) {
+ BlockletSerializer serializer = new BlockletSerializer();
+ String writePath =
+ identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + dataMapName;
+ if (dataMapFactory.getDataMapType() == DataMapType.FG) {
+ FileFactory.mkdirs(writePath, FileFactory.getFileType(writePath));
+ }
+ for (Blocklet blocklet : blocklets) {
ExtendedBlocklet detailedBlocklet =
blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegmentId());
+ if (dataMapFactory.getDataMapType() == DataMapType.FG) {
+ String blockletwritePath =
+ writePath + CarbonCommonConstants.FILE_SEPARATOR + System.nanoTime();
+ detailedBlocklet.setDataMapWriterPath(blockletwritePath);
+ serializer.serializeBlocklet((FineGrainBlocklet) blocklet, blockletwritePath);
+ }
detailedBlocklet.setSegmentId(distributable.getSegmentId());
detailedBlocklets.add(detailedBlocklet);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
new file mode 100644
index 0000000..bcc9bad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+/**
+ * Data Map writer
+ */
+public abstract class AbstractDataMapWriter {
+
+ protected AbsoluteTableIdentifier identifier;
+
+ protected String segmentId;
+
+ protected String writeDirectoryPath;
+
+ public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ String writeDirectoryPath) {
+ this.identifier = identifier;
+ this.segmentId = segmentId;
+ this.writeDirectoryPath = writeDirectoryPath;
+ }
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ public abstract void onBlockStart(String blockId);
+
+ /**
+ * End of block notification
+ */
+ public abstract void onBlockEnd(String blockId);
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ public abstract void onBlockletStart(int blockletId);
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ public abstract void onBlockletEnd(int blockletId);
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ public abstract void onPageAdded(int blockletId, int pageId, ColumnPage[] pages);
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ public abstract void finish() throws IOException;
+
+ /**
+ * It copies the file from temp folder to actual folder
+ *
+ * @param dataMapFile
+ * @throws IOException
+ */
+ protected void commitFile(String dataMapFile) throws IOException {
+ if (!dataMapFile.startsWith(writeDirectoryPath)) {
+ throw new UnsupportedOperationException(
+ "Datamap file " + dataMapFile + " is not written in provided directory path "
+ + writeDirectoryPath);
+ }
+ String dataMapFileName =
+ dataMapFile.substring(writeDirectoryPath.length(), dataMapFile.length());
+ String carbonFilePath = dataMapFileName.substring(0, dataMapFileName.lastIndexOf("/"));
+ String segmentPath = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
+ if (carbonFilePath.length() > 0) {
+ carbonFilePath = segmentPath + carbonFilePath;
+ FileFactory.mkdirs(carbonFilePath, FileFactory.getFileType(carbonFilePath));
+ } else {
+ carbonFilePath = segmentPath;
+ }
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(dataMapFile, carbonFilePath, 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
new file mode 100644
index 0000000..3d4c717
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
+
+public class BlockletSerializer {
+
+ /**
+ * Serialize and write blocklet to the file.
+ * @param grainBlocklet
+ * @param writePath
+ * @throws IOException
+ */
+ public void serializeBlocklet(FineGrainBlocklet grainBlocklet, String writePath)
+ throws IOException {
+ DataOutputStream dataOutputStream =
+ FileFactory.getDataOutputStream(writePath, FileFactory.getFileType(writePath));
+ grainBlocklet.write(dataOutputStream);
+ dataOutputStream.close();
+ }
+
+ /**
+ * Read data from filepath and deserialize blocklet.
+ * @param writePath
+ * @return
+ * @throws IOException
+ */
+ public FineGrainBlocklet deserializeBlocklet(String writePath) throws IOException {
+ DataInputStream inputStream =
+ FileFactory.getDataInputStream(writePath, FileFactory.getFileType(writePath));
+ FineGrainBlocklet blocklet = new FineGrainBlocklet();
+ blocklet.readFields(inputStream);
+ inputStream.close();
+ return blocklet;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index dfe97e3..3fa6d75 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
/**
* Datamap is an entity which can store and retrieve index data.
*/
-public interface DataMap {
+public interface DataMap<T extends Blocklet> {
/**
* It is called to load the data map to memory or to initialize it.
@@ -41,7 +41,7 @@ public interface DataMap {
* @param filterExp
* @return
*/
- List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
List<String> partitions);
// TODO Move this method to Abstract class
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index f5a7404..e900f8a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -21,13 +21,14 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.events.Event;
/**
* Interface for datamap factory, it is responsible for creating the datamap.
*/
-public interface DataMapFactory {
+public interface DataMapFactory<T extends DataMap> {
/**
* Initialization of Datamap factory with the identifier and datamap name
@@ -37,17 +38,17 @@ public interface DataMapFactory {
/**
* Return a new write for this datamap
*/
- DataMapWriter createWriter(String segmentId);
+ AbstractDataMapWriter createWriter(String segmentId, String writeDirectoryPath);
/**
* Get the datamap for segmentid
*/
- List<DataMap> getDataMaps(String segmentId) throws IOException;
+ List<T> getDataMaps(String segmentId) throws IOException;
/**
* Get datamaps for distributable object.
*/
- List<DataMap> getDataMaps(DataMapDistributable distributable) throws IOException;
+ List<T> getDataMaps(DataMapDistributable distributable) throws IOException;
/**
* Get all distributable objects of a segmentid
@@ -75,4 +76,9 @@ public interface DataMapFactory {
* Return metadata of this datamap
*/
DataMapMeta getMeta();
+
+ /**
+ * Type of datamap whether it is FG or CG
+ */
+ DataMapType getDataMapType();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
deleted file mode 100644
index 413eaa5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev;
-
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-
-/**
- * Data Map writer
- */
-public interface DataMapWriter {
-
- /**
- * Start of new block notification.
- * @param blockId file name of the carbondata file
- */
- void onBlockStart(String blockId, String blockPath);
-
- /**
- * End of block notification
- */
- void onBlockEnd(String blockId);
-
- /**
- * Start of new blocklet notification.
- * @param blockletId sequence number of blocklet in the block
- */
- void onBlockletStart(int blockletId);
-
- /**
- * End of blocklet notification
- * @param blockletId sequence number of blocklet in the block
- */
- void onBlockletEnd(int blockletId);
- /**
- * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
- * DataMapMeta returned in DataMapFactory.
- *
- * Implementation should copy the content of `pages` as needed, because `pages` memory
- * may be freed after this method returns, if using unsafe column page.
- */
- void onPageAdded(int blockletId, int pageId, ColumnPage[] pages);
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
new file mode 100644
index 0000000..d79d0c6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.indexstore.Blocklet;
+
+public abstract class AbstractCoarseGrainDataMap implements DataMap<Blocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
new file mode 100644
index 0000000..9789992
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+
+/**
+ * 1. Any filter query which hits the table with datamap will call prune method of CGdatamap.
+ * 2. The prune method of CGDatamap return list Blocklet , these blocklets contain the
+ * information of block and blocklet.
+ * 3. Based on the splits scanrdd schedule the tasks.
+ */
+public abstract class AbstractCoarseGrainDataMapFactory
+ implements DataMapFactory<AbstractCoarseGrainDataMap> {
+
+ @Override public DataMapType getDataMapType() {
+ return DataMapType.CG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
new file mode 100644
index 0000000..310fb3b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
+
+public abstract class AbstractFineGrainDataMap implements DataMap<FineGrainBlocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
new file mode 100644
index 0000000..1ca7fc3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+
+/**
+ * 1. Any filter query which hits the table with datamap will call prune method of FGdatamap.
+ * 2. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the
+ * information of block, blocklet, page and rowids information as well.
+ * 3. The pruned blocklets are internally wriitten to file and returns only the block ,
+ * blocklet and filepath information as part of Splits.
+ * 4. Based on the splits scanrdd schedule the tasks.
+ * 5. In filterscanner we check the datamapwriterpath from split and reNoteads the
+ * bitset if exists. And pass this bitset as input to it.
+ */
+public abstract class AbstractFineGrainDataMapFactory
+ implements DataMapFactory<AbstractFineGrainDataMap> {
+
+ @Override public DataMapType getDataMapType() {
+ return DataMapType.FG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 273f833..df0896a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* Interface data block reference
@@ -119,4 +120,9 @@ public interface DataRefNode {
*/
MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex) throws IOException;
+ /**
+ * Return the indexed data if it has any from disk which was stored by FG datamap.
+ * @return
+ */
+ BitSetGroup getIndexedData();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index c3cc551..907708c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -90,6 +90,8 @@ public class TableBlockInfo implements Distributable, Serializable {
private BlockletDetailInfo detailInfo;
+ private String dataMapWriterPath;
+
public TableBlockInfo(String filePath, long blockOffset, String segmentId,
String[] locations, long blockLength, ColumnarFormatVersion version,
String[] deletedDeltaFilePath) {
@@ -410,4 +412,12 @@ public class TableBlockInfo implements Distributable, Serializable {
public void setBlockletId(String blockletId) {
this.blockletId = blockletId;
}
+
+ public String getDataMapWriterPath() {
+ return dataMapWriterPath;
+ }
+
+ public void setDataMapWriterPath(String dataMapWriterPath) {
+ this.dataMapWriterPath = dataMapWriterPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
index fe4cf83..f5a751b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -23,6 +23,7 @@ import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* Non leaf node abstract class
@@ -222,4 +223,8 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
public int getPageRowCount(int pageNumber) {
throw new UnsupportedOperationException("Unsupported operation");
}
+
+ @Override public BitSetGroup getIndexedData() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
index c200f8d..a6eb695 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* No leaf node of a b+tree class which will keep the matadata(start key) of the
@@ -227,6 +228,10 @@ public class BTreeNonLeafNode implements BTreeNode {
throw new UnsupportedOperationException("Unsupported operation");
}
+ public BitSetGroup getIndexedData() {
+ return null;
+ }
+
/**
* number of pages in blocklet
* @return
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
index d84f3f6..c731e07 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
@@ -16,28 +16,46 @@
*/
package org.apache.carbondata.core.indexstore;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.io.Serializable;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+
/**
* Blocklet
*/
-public class Blocklet implements Serializable {
+public class Blocklet implements Writable,Serializable {
- private String path;
+ private String blockId;
private String blockletId;
- public Blocklet(String path, String blockletId) {
- this.path = path;
+ public Blocklet(String blockId, String blockletId) {
+ this.blockId = blockId;
this.blockletId = blockletId;
}
- public String getPath() {
- return path;
+ // For serialization purpose
+ public Blocklet() {
}
public String getBlockletId() {
return blockletId;
}
+ public String getBlockId() {
+ return blockId;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeUTF(blockId);
+ out.writeUTF(blockletId);
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ blockId = in.readUTF();
+ blockletId = in.readUTF();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index 111a7a2..7598961 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -22,7 +22,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -104,7 +103,6 @@ public class BlockletDataMapIndexStore
List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException {
List<BlockletDataMap> blockletDataMaps = new ArrayList<>(tableSegmentUniqueIdentifiers.size());
List<TableBlockIndexUniqueIdentifier> missedIdentifiers = new ArrayList<>();
- ExecutorService service = null;
// Get the datamaps for each indexfile from cache.
try {
for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) {
@@ -151,10 +149,6 @@ public class BlockletDataMapIndexStore
dataMap.clear();
}
throw new IOException("Problem in loading segment blocks.", e);
- } finally {
- if (service != null) {
- service.shutdownNow();
- }
}
return blockletDataMaps;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
index 21ecba1..3ed826a 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
@@ -44,4 +44,12 @@ public interface BlockletDetailsFetcher {
* @throws IOException
*/
ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, String segmentId) throws IOException;
+
+ /**
+ * Get all the blocklets in a segment
+ *
+ * @param segmentId
+ * @return
+ */
+ List<Blocklet> getAllBlocklets(String segmentId, List<String> partitions) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
index d1bfa35..58a9344 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
@@ -29,8 +29,13 @@ public class ExtendedBlocklet extends Blocklet {
private String[] location;
+ private String path;
+
+ private String dataMapWriterPath;
+
public ExtendedBlocklet(String path, String blockletId) {
super(path, blockletId);
+ this.path = path;
}
public BlockletDetailInfo getDetailInfo() {
@@ -60,4 +65,16 @@ public class ExtendedBlocklet extends Blocklet {
public void setSegmentId(String segmentId) {
this.segmentId = segmentId;
}
+
+ public String getPath() {
+ return path;
+ }
+
+ public String getDataMapWriterPath() {
+ return dataMapWriterPath;
+ }
+
+ public void setDataMapWriterPath(String dataMapWriterPath) {
+ this.dataMapWriterPath = dataMapWriterPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
new file mode 100644
index 0000000..266120e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+/**
+ * FineGrainBlocklet
+ */
+public class FineGrainBlocklet extends Blocklet implements Serializable {
+
+ private List<Page> pages;
+
+ public FineGrainBlocklet(String blockId, String blockletId, List<Page> pages) {
+ super(blockId, blockletId);
+ this.pages = pages;
+ }
+
+ // For serialization purpose
+ public FineGrainBlocklet() {
+
+ }
+
+ public List<Page> getPages() {
+ return pages;
+ }
+
+ public static class Page implements Writable,Serializable {
+
+ private int pageId;
+
+ private int[] rowId;
+
+ public BitSet getBitSet() {
+ BitSet bitSet =
+ new BitSet(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
+ for (int row : rowId) {
+ bitSet.set(row);
+ }
+ return bitSet;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeInt(pageId);
+ out.writeInt(rowId.length);
+ for (int i = 0; i < rowId.length; i++) {
+ out.writeInt(rowId[i]);
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ pageId = in.readInt();
+ int length = in.readInt();
+ rowId = new int[length];
+ for (int i = 0; i < length; i++) {
+ rowId[i] = in.readInt();
+ }
+ }
+
+ public void setPageId(int pageId) {
+ this.pageId = pageId;
+ }
+
+ public void setRowId(int[] rowId) {
+ this.rowId = rowId;
+ }
+ }
+
+ public BitSetGroup getBitSetGroup(int numberOfPages) {
+ BitSetGroup bitSetGroup = new BitSetGroup(numberOfPages);
+ for (int i = 0; i < pages.size(); i++) {
+ bitSetGroup.setBitSet(pages.get(i).getBitSet(), pages.get(i).pageId);
+ }
+ return bitSetGroup;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ super.write(out);
+ int size = pages.size();
+ out.writeInt(size);
+ for (Page page : pages) {
+ page.write(out);
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ pages = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ Page page = new Page();
+ page.readFields(in);
+ pages.add(page);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index d331c2b..99a47ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -35,8 +35,8 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cacheable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -73,7 +73,7 @@ import org.xerial.snappy.Snappy;
/**
* Datamap implementation for blocklet.
*/
-public class BlockletDataMap implements DataMap, Cacheable {
+public class BlockletDataMap extends AbstractCoarseGrainDataMap implements Cacheable {
private static final LogService LOGGER =
LogServiceFactory.getLogService(BlockletDataMap.class.getName());
@@ -609,9 +609,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
for (int i = 0; i < unsafeMemorySummaryDMStore.getRowCount(); i++) {
DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(i);
- boolean isScanRequired = FilterExpressionProcessor
- .isScanRequired(filterExecuter, getMinMaxValue(unsafeRow, TASK_MAX_VALUES_INDEX),
- getMinMaxValue(unsafeRow, TASK_MIN_VALUES_INDEX));
+ boolean isScanRequired = FilterExpressionProcessor.isScanRequired(
+ filterExecuter, getMinMaxValue(unsafeRow, TASK_MAX_VALUES_INDEX),
+ getMinMaxValue(unsafeRow, TASK_MIN_VALUES_INDEX));
if (isScanRequired) {
return true;
}
@@ -684,7 +684,6 @@ public class BlockletDataMap implements DataMap, Cacheable {
startIndex++;
}
}
-
return blocklets;
}
@@ -977,4 +976,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
return memoryUsed;
}
+ public SegmentProperties getSegmentProperties() {
+ return segmentProperties;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 61e5ceb..f6b8165 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -27,9 +27,10 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -39,9 +40,6 @@ import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.DataFileFooterConverter;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -53,7 +51,8 @@ import org.apache.hadoop.fs.RemoteIterator;
/**
* Table map for blocklet
*/
-public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFetcher,
+public class BlockletDataMapFactory extends AbstractCoarseGrainDataMapFactory
+ implements BlockletDetailsFetcher,
SegmentPropertiesFetcher {
private AbsoluteTableIdentifier identifier;
@@ -61,10 +60,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
// segmentId -> list of index file
private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
- // segmentId -> SegmentProperties.
- private Map<String, SegmentProperties> segmentPropertiesMap = new HashMap<>();
-
- private Cache<TableBlockIndexUniqueIdentifier, DataMap> cache;
+ private Cache<TableBlockIndexUniqueIdentifier, AbstractCoarseGrainDataMap> cache;
@Override
public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
@@ -74,12 +70,12 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override
- public DataMapWriter createWriter(String segmentId) {
+ public AbstractDataMapWriter createWriter(String segmentId, String dataWriterPath) {
throw new UnsupportedOperationException("not implemented");
}
@Override
- public List<DataMap> getDataMaps(String segmentId) throws IOException {
+ public List<AbstractCoarseGrainDataMap> getDataMaps(String segmentId) throws IOException {
List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
getTableBlockIndexUniqueIdentifiers(segmentId);
return cache.getAll(tableBlockIndexUniqueIdentifiers);
@@ -140,17 +136,18 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
private ExtendedBlocklet getExtendedBlocklet(List<TableBlockIndexUniqueIdentifier> identifiers,
Blocklet blocklet) throws IOException {
- String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getPath());
+ String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getBlockId());
for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
if (identifier.getCarbonIndexFileName().equals(carbonIndexFileName)) {
DataMap dataMap = cache.get(identifier);
return ((BlockletDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId());
}
}
- throw new IOException("Blocklet with blockid " + blocklet.getPath() + " not found ");
+ throw new IOException("Blocklet with blockid " + blocklet.getBlockletId() + " not found ");
}
+
@Override
public List<DataMapDistributable> toDistributable(String segmentId) {
CarbonFile[] carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentId);
@@ -179,7 +176,6 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
@Override
public void clear(String segmentId) {
- segmentPropertiesMap.remove(segmentId);
List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segmentId);
if (blockIndexes != null) {
for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
@@ -200,7 +196,8 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override
- public List<DataMap> getDataMaps(DataMapDistributable distributable) throws IOException {
+ public List<AbstractCoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
if (mapDistributable.getFilePath().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
@@ -217,7 +214,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
indexFile));
}
}
- List<DataMap> dataMaps;
+ List<AbstractCoarseGrainDataMap> dataMaps;
try {
dataMaps = cache.getAll(identifiers);
} catch (IOException e) {
@@ -233,23 +230,21 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override public SegmentProperties getSegmentProperties(String segmentId) throws IOException {
- SegmentProperties segmentProperties = segmentPropertiesMap.get(segmentId);
- if (segmentProperties == null) {
- int[] columnCardinality;
- List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
- getTableBlockIndexUniqueIdentifiers(segmentId);
- DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
- List<DataFileFooter> indexInfo =
- fileFooterConverter.getIndexInfo(tableBlockIndexUniqueIdentifiers.get(0).getFilePath());
- for (DataFileFooter fileFooter : indexInfo) {
- List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
- if (segmentProperties == null) {
- columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
- segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
- }
- }
- segmentPropertiesMap.put(segmentId, segmentProperties);
+ List<AbstractCoarseGrainDataMap> dataMaps = getDataMaps(segmentId);
+ assert (dataMaps.size() > 0);
+ AbstractCoarseGrainDataMap coarseGrainDataMap = dataMaps.get(0);
+ assert (coarseGrainDataMap instanceof BlockletDataMap);
+ BlockletDataMap dataMap = (BlockletDataMap) coarseGrainDataMap;
+ return dataMap.getSegmentProperties();
+ }
+
+ @Override public List<Blocklet> getAllBlocklets(String segmentId, List<String> partitions)
+ throws IOException {
+ List<Blocklet> blocklets = new ArrayList<>();
+ List<AbstractCoarseGrainDataMap> dataMaps = getDataMaps(segmentId);
+ for (AbstractCoarseGrainDataMap dataMap : dataMaps) {
+ blocklets.addAll(dataMap.prune(null, getSegmentProperties(segmentId), partitions));
}
- return segmentProperties;
+ return blocklets;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
index 4d10fd6..ad4a7cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -29,7 +30,9 @@ import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
/**
@@ -43,6 +46,8 @@ public class BlockletDataRefNode implements DataRefNode {
private int[] dimensionLens;
+ private BlockletSerializer blockletSerializer;
+
BlockletDataRefNode(List<TableBlockInfo> blockInfos, int index, int[] dimensionLens) {
this.blockInfos = blockInfos;
// Update row count and page count to blocklet info
@@ -67,6 +72,7 @@ public class BlockletDataRefNode implements DataRefNode {
}
this.index = index;
this.dimensionLens = dimensionLens;
+ this.blockletSerializer = new BlockletSerializer();
}
@Override public DataRefNode getNextDataRefNode() {
@@ -165,11 +171,28 @@ public class BlockletDataRefNode implements DataRefNode {
}
}
- @Override public int numberOfPages() {
+ @Override
+ public int numberOfPages() {
return blockInfos.get(index).getDetailInfo().getPagesCount();
}
- @Override public int getPageRowCount(int pageNumber) {
+ @Override
+ public BitSetGroup getIndexedData() {
+ String dataMapWriterPath = blockInfos.get(index).getDataMapWriterPath();
+ if (dataMapWriterPath != null) {
+ try {
+ FineGrainBlocklet blocklet = blockletSerializer.deserializeBlocklet(dataMapWriterPath);
+ return blocklet.getBitSetGroup(numberOfPages());
+ } catch (IOException e) {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public int getPageRowCount(int pageNumber) {
return blockInfos.get(index).getDetailInfo().getBlockletInfo()
.getNumberOfRowsPerPage()[pageNumber];
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
index a30f64c..95232e5 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
@@ -21,6 +21,8 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
/**
@@ -29,7 +31,10 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
*/
public class IndexWrapper extends AbstractIndex {
+ private List<TableBlockInfo> blockInfos;
+
public IndexWrapper(List<TableBlockInfo> blockInfos) {
+ this.blockInfos = blockInfos;
segmentProperties = new SegmentProperties(blockInfos.get(0).getDetailInfo().getColumnSchemas(),
blockInfos.get(0).getDetailInfo().getDimLens());
dataRefNode = new BlockletDataRefNode(blockInfos, 0,
@@ -38,4 +43,17 @@ public class IndexWrapper extends AbstractIndex {
@Override public void buildIndex(List<DataFileFooter> footerList) {
}
+
+ @Override public void clear() {
+ super.clear();
+ if (blockInfos != null) {
+ for (TableBlockInfo blockInfo : blockInfos) {
+ String dataMapWriterPath = blockInfo.getDataMapWriterPath();
+ if (dataMapWriterPath != null) {
+ CarbonFile file = FileFactory.getCarbonFile(dataMapWriterPath);
+ FileFactory.deleteAllCarbonFilesOfDir(file);
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
deleted file mode 100644
index 9d77010..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore.schema;
-
-/**
- * Types of filters of select query
- */
-public enum FilterType {
- EQUALTO, GREATER_THAN, LESS_THAN, GREATER_THAN_EQUAL, LESS_THAN_EQUAL, LIKE
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 5974666..26b0f67 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -314,6 +314,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
BitSetGroup prvBitSetGroup, int pageNumber) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ if (prvPageBitSet == null || prvPageBitSet.isEmpty()) {
+ return prvPageBitSet;
+ }
BitSet bitSet = new BitSet();
bitSet.or(prvPageBitSet);
byte[][] filterKeys = dimColumnExecuterInfo.getExcludeFilterKeys();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 05328f3..516ed41 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -334,6 +334,9 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ if (prvPageBitSet == null || prvPageBitSet.isEmpty()) {
+ return prvPageBitSet;
+ }
BitSet bitSet = new BitSet(numberOfRows);
byte[][] filterKeys = dimColumnExecuterInfo.getFilterKeys();
int compareResult = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
index 1c73d63..033c3dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
@@ -163,6 +163,8 @@ public class BlockletFilterScanner extends BlockletFullScanner {
.get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
totalBlockletStatistic.getCount() + 1);
+ // set the indexed data if it has any during fgdatamap pruning.
+ rawBlockletColumnChunks.setBitSetGroup(rawBlockletColumnChunks.getDataBlock().getIndexedData());
// apply filter on actual data, for each page
BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(rawBlockletColumnChunks,
useBitSetPipeLine);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 80a382c..c8b36e2 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -55,6 +56,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
@@ -99,6 +101,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.thrift.TBase;
import org.apache.thrift.TException;
@@ -131,6 +134,13 @@ public final class CarbonUtil {
private static final Configuration conf = new Configuration(true);
+ /**
+ * dfs.bytes-per-checksum
+ * HDFS checksum length, block size for a file should be exactly divisible
+ * by this value
+ */
+ private static final int HDFS_CHECKSUM_LENGTH = 512;
+
private CarbonUtil() {
}
@@ -2356,5 +2366,92 @@ public final class CarbonUtil {
return Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
}
+
+ /**
+ * This method will copy the given file to carbon store location
+ *
+ * @param localFilePath local file name with full path
+ * @throws CarbonDataWriterException
+ */
+ public static void copyCarbonDataFileToCarbonStorePath(String localFilePath,
+ String carbonDataDirectoryPath, long fileSizeInBytes)
+ throws CarbonDataWriterException {
+ long copyStartTime = System.currentTimeMillis();
+ LOGGER.info("Copying " + localFilePath + " --> " + carbonDataDirectoryPath);
+ try {
+ CarbonFile localCarbonFile =
+ FileFactory.getCarbonFile(localFilePath, FileFactory.getFileType(localFilePath));
+ String carbonFilePath = carbonDataDirectoryPath + localFilePath
+ .substring(localFilePath.lastIndexOf(File.separator));
+ copyLocalFileToCarbonStore(carbonFilePath, localFilePath,
+ CarbonCommonConstants.BYTEBUFFER_SIZE,
+ getMaxOfBlockAndFileSize(fileSizeInBytes, localCarbonFile.getSize()));
+ } catch (IOException e) {
+ throw new CarbonDataWriterException(
+ "Problem while copying file from local store to carbon store", e);
+ }
+ LOGGER.info(
+ "Total copy time (ms) to copy file " + localFilePath + " is " + (System.currentTimeMillis()
+ - copyStartTime));
+ }
+
+ /**
+ * This method will read the local carbon data file and write to carbon data file in HDFS
+ *
+ * @param carbonStoreFilePath
+ * @param localFilePath
+ * @param bufferSize
+ * @param blockSize
+ * @throws IOException
+ */
+ private static void copyLocalFileToCarbonStore(String carbonStoreFilePath, String localFilePath,
+ int bufferSize, long blockSize) throws IOException {
+ DataOutputStream dataOutputStream = null;
+ DataInputStream dataInputStream = null;
+ try {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize
+ + " (bytes");
+ }
+ dataOutputStream = FileFactory
+ .getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
+ bufferSize, blockSize);
+ dataInputStream = FileFactory
+ .getDataInputStream(localFilePath, FileFactory.getFileType(localFilePath), bufferSize);
+ IOUtils.copyBytes(dataInputStream, dataOutputStream, bufferSize);
+ } finally {
+ CarbonUtil.closeStream(dataInputStream);
+ CarbonUtil.closeStream(dataOutputStream);
+ }
+ }
+
+ /**
+ * This method will return max of block size and file size
+ *
+ * @param blockSize
+ * @param fileSize
+ * @return
+ */
+ private static long getMaxOfBlockAndFileSize(long blockSize, long fileSize) {
+ long maxSize = blockSize;
+ if (fileSize > blockSize) {
+ maxSize = fileSize;
+ }
+ // block size should be exactly divisible by 512 which is maintained by HDFS as bytes
+ // per checksum, dfs.bytes-per-checksum=512 must divide block size
+ long remainder = maxSize % HDFS_CHECKSUM_LENGTH;
+ if (remainder > 0) {
+ maxSize = maxSize + HDFS_CHECKSUM_LENGTH - remainder;
+ }
+ // convert to make block size more readable.
+ String readableBlockSize = ByteUtil.convertByteToReadable(blockSize);
+ String readableFileSize = ByteUtil.convertByteToReadable(fileSize);
+ String readableMaxSize = ByteUtil.convertByteToReadable(maxSize);
+ LOGGER.info(
+ "The configured block size is " + readableBlockSize + ", the actual carbon file size is "
+ + readableFileSize + ", choose the max value " + readableMaxSize
+ + " as the block size on HDFS");
+ return maxSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
index 2ad6327..8002e57 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
@@ -28,7 +28,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
@@ -36,7 +37,9 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.row.DataMapRow;
import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -47,7 +50,7 @@ import com.google.gson.Gson;
/**
* Datamap implementation for min max blocklet.
*/
-public class MinMaxDataMap implements DataMap {
+public class MinMaxDataMap extends AbstractCoarseGrainDataMap {
private static final LogService LOGGER =
LogServiceFactory.getLogService(MinMaxDataMap.class.getName());
@@ -58,8 +61,9 @@ public class MinMaxDataMap implements DataMap {
private MinMaxIndexBlockDetails[] readMinMaxDataMap;
- @Override public void init(String filePath) throws MemoryException, IOException {
- this.filePath = filePath;
+ @Override
+ public void init(DataMapModel model) throws MemoryException, IOException {
+ this.filePath = model.getFilePath();
CarbonFile[] listFiles = getCarbonMinMaxIndexFiles(filePath, "0");
for (int i = 0; i < listFiles.length; i++) {
readMinMaxDataMap = readJson(listFiles[i].getPath());
@@ -76,7 +80,7 @@ public class MinMaxDataMap implements DataMap {
});
}
- public MinMaxIndexBlockDetails[] readJson(String filePath) throws IOException {
+ private MinMaxIndexBlockDetails[] readJson(String filePath) {
Gson gsonObjectToRead = new Gson();
DataInputStream dataInputStream = null;
BufferedReader buffReader = null;
@@ -90,8 +94,7 @@ public class MinMaxDataMap implements DataMap {
return null;
}
dataInputStream = fileOperation.openForRead();
- inStream = new InputStreamReader(dataInputStream,
- CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
+ inStream = new InputStreamReader(dataInputStream, "UTF-8");
buffReader = new BufferedReader(inStream);
readMinMax = gsonObjectToRead.fromJson(buffReader, MinMaxIndexBlockDetails[].class);
} catch (IOException e) {
@@ -109,14 +112,14 @@ public class MinMaxDataMap implements DataMap {
* @param segmentProperties
* @return
*/
- @Override public List<Blocklet> prune(FilterResolverIntf filterExp,
- SegmentProperties segmentProperties) {
+ @Override
+ public List<Blocklet> prune(FilterResolverIntf filterExp,
+ SegmentProperties segmentProperties, List<String> partitions) {
List<Blocklet> blocklets = new ArrayList<>();
if (filterExp == null) {
for (int i = 0; i < readMinMaxDataMap.length; i++) {
- blocklets.add(new Blocklet(readMinMaxDataMap[i].getFilePath(),
- String.valueOf(readMinMaxDataMap[i].getBlockletId())));
+ blocklets.add(new Blocklet(filePath, String.valueOf(readMinMaxDataMap[i].getBlockletId())));
}
} else {
FilterExecuter filterExecuter =
@@ -126,7 +129,7 @@ public class MinMaxDataMap implements DataMap {
BitSet bitSet = filterExecuter.isScanRequired(readMinMaxDataMap[startIndex].getMaxValues(),
readMinMaxDataMap[startIndex].getMinValues());
if (!bitSet.isEmpty()) {
- blocklets.add(new Blocklet(readMinMaxDataMap[startIndex].getFilePath(),
+ blocklets.add(new Blocklet(filePath,
String.valueOf(readMinMaxDataMap[startIndex].getBlockletId())));
}
startIndex++;
@@ -136,6 +139,11 @@ public class MinMaxDataMap implements DataMap {
}
@Override
+ public boolean isScanRequired(FilterResolverIntf filterExp) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void clear() {
readMinMaxDataMap = null;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e972fd3d/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
index b196d0d..5203cb3 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -25,49 +25,51 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
-import org.apache.carbondata.core.events.ChangeEvent;
-import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.events.Event;
/**
* Min Max DataMap Factory
*/
-public class MinMaxDataMapFactory implements DataMapFactory {
+public class MinMaxDataMapFactory extends AbstractCoarseGrainDataMapFactory {
private AbsoluteTableIdentifier identifier;
- @Override
- public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ @Override public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
this.identifier = identifier;
}
/**
* createWriter will return the MinMaxDataWriter.
+ *
* @param segmentId
* @return
*/
- @Override
- public DataMapWriter createWriter(String segmentId) {
- return new MinMaxDataWriter();
+ @Override public AbstractDataMapWriter createWriter(String segmentId, String dataWritePath) {
+ return new MinMaxDataWriter(identifier, segmentId, dataWritePath);
}
/**
* getDataMaps Factory method Initializes the Min Max Data Map and returns.
+ *
* @param segmentId
* @return
* @throws IOException
*/
- @Override public List<DataMap> getDataMaps(String segmentId) throws IOException {
- List<DataMap> dataMapList = new ArrayList<>();
+ @Override public List<AbstractCoarseGrainDataMap> getDataMaps(String segmentId)
+ throws IOException {
+ List<AbstractCoarseGrainDataMap> dataMapList = new ArrayList<>();
// Form a dataMap of Type MinMaxDataMap.
MinMaxDataMap dataMap = new MinMaxDataMap();
try {
- dataMap.init(identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + File.separator);
+ dataMap.init(new DataMapModel(
+ identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + File.separator));
} catch (MemoryException ex) {
}
@@ -76,7 +78,6 @@ public class MinMaxDataMapFactory implements DataMapFactory {
}
/**
- *
* @param segmentId
* @return
*/
@@ -86,6 +87,7 @@ public class MinMaxDataMapFactory implements DataMapFactory {
/**
* Clear the DataMap.
+ *
* @param segmentId
*/
@Override public void clear(String segmentId) {
@@ -94,21 +96,20 @@ public class MinMaxDataMapFactory implements DataMapFactory {
/**
* Clearing the data map.
*/
- @Override
- public void clear() {
+ @Override public void clear() {
}
- @Override public DataMap getDataMap(DataMapDistributable distributable) {
+ @Override public List<AbstractCoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
return null;
}
- @Override
- public void fireEvent(ChangeEvent event) {
+ @Override public void fireEvent(Event event) {
}
- @Override
- public DataMapMeta getMeta() {
- return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")), FilterType.EQUALTO);
+ @Override public DataMapMeta getMeta() {
+ return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")),
+ new ArrayList<ExpressionType>());
}
}
\ No newline at end of file
[36/50] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 224a69f..c18f090 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
@@ -58,7 +58,7 @@ import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -67,20 +67,20 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(RowLevelFilterExecuterImpl.class.getName());
- protected List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
- protected List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
+ List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
+ List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
protected Expression exp;
protected AbsoluteTableIdentifier tableIdentifier;
protected SegmentProperties segmentProperties;
/**
* it has index at which given dimension is stored in file
*/
- protected int[] dimensionBlocksIndex;
+ int[] dimensionChunkIndex;
/**
* it has index at which given measure is stored in file
*/
- protected int[] measureBlocksIndex;
+ int[] measureChunkIndex;
private Map<Integer, GenericQueryType> complexDimensionInfoMap;
@@ -88,18 +88,18 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* flag to check whether the filter dimension is present in current block list of dimensions.
* Applicable for restructure scenarios
*/
- protected boolean[] isDimensionPresentInCurrentBlock;
+ boolean[] isDimensionPresentInCurrentBlock;
/**
* flag to check whether the filter measure is present in current block list of measures.
* Applicable for restructure scenarios
*/
- protected boolean[] isMeasurePresentInCurrentBlock;
+ boolean[] isMeasurePresentInCurrentBlock;
/**
* is dimension column data is natural sorted
*/
- protected boolean isNaturalSorted;
+ boolean isNaturalSorted;
/**
* date direct dictionary generator
@@ -123,10 +123,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
if (this.dimColEvaluatorInfoList.size() > 0) {
this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()];
- this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
+ this.dimensionChunkIndex = new int[dimColEvaluatorInfoList.size()];
} else {
this.isDimensionPresentInCurrentBlock = new boolean[]{false};
- this.dimensionBlocksIndex = new int[]{0};
+ this.dimensionChunkIndex = new int[]{0};
}
if (null == msrColEvalutorInfoList) {
this.msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(20);
@@ -135,10 +135,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
if (this.msrColEvalutorInfoList.size() > 0) {
this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()];
- this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
+ this.measureChunkIndex = new int[msrColEvalutorInfoList.size()];
} else {
this.isMeasurePresentInCurrentBlock = new boolean[]{false};
- this.measureBlocksIndex = new int[] {0};
+ this.measureChunkIndex = new int[] {0};
}
this.exp = exp;
this.tableIdentifier = tableIdentifier;
@@ -147,22 +147,22 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.DATE);
this.timestampDictionaryGenerator =
DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.TIMESTAMP);
- initDimensionBlockIndexes();
- initMeasureBlockIndexes();
+ initDimensionChunkIndexes();
+ initMeasureChunkIndexes();
}
/**
* This method will initialize the dimension info for the current block to be
* used for filtering the data
*/
- private void initDimensionBlockIndexes() {
+ private void initDimensionChunkIndexes() {
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
// find the dimension in the current block dimensions list
CarbonDimension dimensionFromCurrentBlock = segmentProperties
.getDimensionFromCurrentBlock(dimColEvaluatorInfoList.get(i).getDimension());
if (null != dimensionFromCurrentBlock) {
dimColEvaluatorInfoList.get(i).setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
- this.dimensionBlocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping()
+ this.dimensionChunkIndex[i] = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimensionFromCurrentBlock.getOrdinal());
isDimensionPresentInCurrentBlock[i] = true;
}
@@ -173,14 +173,14 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* This method will initialize the measure info for the current block to be
* used for filtering the data
*/
- private void initMeasureBlockIndexes() {
+ private void initMeasureChunkIndexes() {
for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
// find the measure in the current block measures list
CarbonMeasure measureFromCurrentBlock = segmentProperties.getMeasureFromCurrentBlock(
msrColEvalutorInfoList.get(i).getCarbonColumn().getColumnId());
if (null != measureFromCurrentBlock) {
msrColEvalutorInfoList.get(i).setColumnIndex(measureFromCurrentBlock.getOrdinal());
- this.measureBlocksIndex[i] = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ this.measureChunkIndex[i] = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(measureFromCurrentBlock.getOrdinal());
isMeasurePresentInCurrentBlock[i] = true;
}
@@ -188,9 +188,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- readBlocks(blockChunkHolder);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ readColumnChunks(rawBlockletColumnChunks);
// CHECKSTYLE:ON
int[] numberOfRows = null;
@@ -198,26 +198,26 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
if (dimColEvaluatorInfoList.size() > 0) {
if (isDimensionPresentInCurrentBlock[0]) {
- pageNumbers =
- blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getPagesCount();
- numberOfRows =
- blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getRowCount();
+ pageNumbers = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]]
+ .getPagesCount();
+ numberOfRows = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]]
+ .getRowCount();
} else {
// specific for restructure case where default values need to be filled
- pageNumbers = blockChunkHolder.getDataBlock().numberOfPages();
- numberOfRows = new int[] { blockChunkHolder.getDataBlock().nodeSize() };
+ pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ numberOfRows = new int[] { rawBlockletColumnChunks.getDataBlock().numRows() };
}
}
if (msrColEvalutorInfoList.size() > 0) {
if (isMeasurePresentInCurrentBlock[0]) {
- pageNumbers =
- blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getPagesCount();
- numberOfRows =
- blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getRowCount();
+ pageNumbers = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]
+ .getPagesCount();
+ numberOfRows = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]
+ .getRowCount();
} else {
// specific for restructure case where default values need to be filled
- pageNumbers = blockChunkHolder.getDataBlock().numberOfPages();
- numberOfRows = new int[] { blockChunkHolder.getDataBlock().nodeSize() };
+ pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ numberOfRows = new int[] { rawBlockletColumnChunks.getDataBlock().numRows() };
}
}
BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
@@ -227,10 +227,12 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
BitSet prvBitset = null;
// if bitset pipe line is enabled then use rowid from previous bitset
// otherwise use older flow
- if (!useBitsetPipeLine || null == blockChunkHolder.getBitSetGroup() || null == bitSetGroup
- .getBitSet(i) || blockChunkHolder.getBitSetGroup().getBitSet(i).isEmpty()) {
+ if (!useBitsetPipeLine ||
+ null == rawBlockletColumnChunks.getBitSetGroup() ||
+ null == bitSetGroup.getBitSet(i) ||
+ rawBlockletColumnChunks.getBitSetGroup().getBitSet(i).isEmpty()) {
for (int index = 0; index < numberOfRows[i]; index++) {
- createRow(blockChunkHolder, row, i, index);
+ createRow(rawBlockletColumnChunks, row, i, index);
Boolean rslt = false;
try {
rslt = exp.evaluate(row).getBoolean();
@@ -246,10 +248,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
}
} else {
- prvBitset = blockChunkHolder.getBitSetGroup().getBitSet(i);
+ prvBitset = rawBlockletColumnChunks.getBitSetGroup().getBitSet(i);
for (int index = prvBitset.nextSetBit(0);
index >= 0; index = prvBitset.nextSetBit(index + 1)) {
- createRow(blockChunkHolder, row, i, index);
+ createRow(rawBlockletColumnChunks, row, i, index);
Boolean rslt = false;
try {
rslt = exp.evaluate(row).getBoolean();
@@ -284,8 +286,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @param index
* @throws IOException
*/
- private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int pageIndex, int index)
- throws IOException {
+ private void createRow(RawBlockletColumnChunks blockChunkHolder, RowIntf row, int pageIndex,
+ int index) throws IOException {
Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
String memberString;
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
@@ -302,14 +304,14 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
record[dimColumnEvaluatorInfo.getRowIndex()] =
dimColumnEvaluatorInfo.getDimension().getDefaultValue();
}
- DimensionColumnDataChunk columnDataChunk =
- blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]
- .convertToDimColDataChunk(pageIndex);
+ DimensionColumnPage columnDataChunk =
+ blockChunkHolder.getDimensionRawColumnChunks()[dimensionChunkIndex[i]]
+ .decodeColumnPage(pageIndex);
if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
- && columnDataChunk instanceof VariableLengthDimensionDataChunk) {
+ && columnDataChunk instanceof VariableLengthDimensionColumnPage) {
- VariableLengthDimensionDataChunk dimensionColumnDataChunk =
- (VariableLengthDimensionDataChunk) columnDataChunk;
+ VariableLengthDimensionColumnPage dimensionColumnDataChunk =
+ (VariableLengthDimensionColumnPage) columnDataChunk;
byte[] memberBytes = dimensionColumnDataChunk.getChunkData(index);
if (null != memberBytes) {
if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
@@ -322,8 +324,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
dimColumnEvaluatorInfo.getDimension().getDataType());
}
} else {
- int dictionaryValue = readSurrogatesFromColumnBlock(blockChunkHolder, index, pageIndex,
- dimColumnEvaluatorInfo, dimensionBlocksIndex[i]);
+ int dictionaryValue = readSurrogatesFromColumnChunk(blockChunkHolder, index, pageIndex,
+ dimColumnEvaluatorInfo, dimensionChunkIndex[i]);
if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
&& !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
memberString =
@@ -341,11 +343,11 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
} else {
try {
- GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
+ GenericQueryType complexType = complexDimensionInfoMap.get(dimensionChunkIndex[i]);
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
complexType.parseBlocksAndReturnComplexColumnByteArray(
- blockChunkHolder.getDimensionRawDataChunk(), index, pageIndex, dataOutputStream);
+ blockChunkHolder.getDimensionRawColumnChunks(), index, pageIndex, dataOutputStream);
record[dimColumnEvaluatorInfo.getRowIndex()] = complexType
.getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
byteStream.close();
@@ -384,8 +386,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
Object msrValue;
ColumnPage columnPage =
- blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]
- .convertToColumnPage(pageIndex);
+ blockChunkHolder.getMeasureRawColumnChunks()[measureChunkIndex[0]]
+ .decodeColumnPage(pageIndex);
if (msrType == DataTypes.BOOLEAN) {
msrValue = columnPage.getBoolean(index);
} else if (msrType == DataTypes.SHORT) {
@@ -487,10 +489,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @param dimColumnEvaluatorInfo
* @return
*/
- private int readSurrogatesFromColumnBlock(BlocksChunkHolder blockChunkHolder, int index, int page,
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
- DimensionColumnDataChunk dataChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex].convertToDimColDataChunk(page);
+ private int readSurrogatesFromColumnChunk(RawBlockletColumnChunks blockChunkHolder, int index,
+ int page, DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int chunkIndex) {
+ DimensionColumnPage dataChunk =
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex].decodeColumnPage(page);
if (dimColumnEvaluatorInfo.getDimension().isColumnar()) {
byte[] rawData = dataChunk.getChunkData(index);
ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
@@ -506,7 +508,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @param dimColumnEvaluatorInfo
* @return read surrogate of given row of given column group dimension
*/
- private int readSurrogatesFromColumnGroupBlock(DimensionColumnDataChunk chunk, int index,
+ private int readSurrogatesFromColumnGroupBlock(DimensionColumnPage chunk, int index,
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
try {
KeyStructureInfo keyStructureInfo =
@@ -514,9 +516,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
byte[] colData = chunk.getChunkData(index);
long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
int colGroupId =
- QueryUtil.getColumnGroupId(segmentProperties, dimensionBlocksIndex[0]);
+ QueryUtil.getColumnGroupId(segmentProperties, dimensionChunkIndex[0]);
return (int) result[segmentProperties
- .getColumnGroupMdKeyOrdinal(colGroupId, dimensionBlocksIndex[0])];
+ .getColumnGroupMdKeyOrdinal(colGroupId, dimensionChunkIndex[0])];
} catch (KeyGenException e) {
LOGGER.error(e);
}
@@ -524,33 +526,36 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
bitSet.set(0);
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) {
- if (null == blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]) {
- blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]] =
- blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), dimensionBlocksIndex[i]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[i]])
+ {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[i]] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), dimensionChunkIndex[i]);
}
} else {
- GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
- complexType.fillRequiredBlockData(blockChunkHolder);
+ GenericQueryType complexType = complexDimensionInfoMap.get(dimensionChunkIndex[i]);
+ complexType.fillRequiredBlockData(rawBlockletColumnChunks);
}
}
if (null != msrColEvalutorInfoList) {
for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
- if (null == blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]) {
- blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]] =
- blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), measureBlocksIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), measureChunkIndex[0]);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 306f3fa..6baa3e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -32,11 +32,10 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -55,12 +54,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
*/
private boolean isDefaultValuePresentInFilter;
private int lastDimensionColOrdinal = 0;
- public RowLevelRangeGrtThanFiterExecuterImpl(
- List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
+
+ RowLevelRangeGrtThanFiterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
- Object[] msrFilterRangeValues,
- SegmentProperties segmentProperties) {
+ Object[] msrFilterRangeValues, SegmentProperties segmentProperties) {
super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
null);
this.filterRangeValues = filterRangeValues;
@@ -110,17 +108,18 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
}
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
boolean isScanRequired = false;
byte[] maxValue = null;
if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
- maxValue = blockMaxValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ maxValue = blockMaxValue[measureChunkIndex[0] + lastDimensionColOrdinal];
isScanRequired =
isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
- maxValue = blockMaxValue[dimensionBlocksIndex[0]];
+ maxValue = blockMaxValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(maxValue, filterRangeValues);
}
} else {
@@ -167,24 +166,25 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
@@ -196,26 +196,28 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
@@ -225,7 +227,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
.getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i],
msrColEvalutorInfoList.get(0).getType()));
ColumnPage columnPage =
- rawColumnChunk.convertToColumnPage(i);
+ rawColumnChunk.decodeColumnPage(i);
if (compare < 0 && columnPage.getNullBits().isEmpty()) {
BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
@@ -238,7 +240,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -278,16 +280,16 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return bitSet;
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = null;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows);
} else {
- bitSet = setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ bitSet = setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows);
}
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet,
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet,
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
}
return bitSet;
@@ -299,12 +301,12 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
int last = 0;
@@ -312,11 +314,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
byte[][] filterValues = this.filterRangeValues;
for (int i = 0; i < filterValues.length; i++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i], true);
if (start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[i],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[i],
numerOfRows);
}
// Logic will handle the case where the range filter member is not present in block
@@ -332,7 +334,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
// index needs to be compared by the filter member if its > filter then from that
// index the bitset will be considered for filtering process.
if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
> 0) {
start = start + 1;
}
@@ -340,7 +342,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
last = start;
for (int j = start; j < numerOfRows; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
last++;
}
startIndex = last;
@@ -358,11 +360,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
* be called if the column is sorted default so column index
* mapping will be present for accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
@@ -373,11 +375,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
int startIndex = 0;
for (int k = 0; k < filterValues.length; k++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, filterValues[k], true);
if (start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[k],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[k],
numerOfRows);
}
if (start < 0) {
@@ -388,7 +390,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its > filter then from that
// index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) > 0) {
+ if (ByteUtil.compare(filterValues[k], dimensionColumnPage.getChunkData(start)) > 0) {
start = start + 1;
}
}
@@ -405,7 +407,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
} else {
for (int k = 0; k < filterValues.length; k++) {
for (int i = 0; i < numerOfRows; i++) {
- if (ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValues[k]) > 0) {
+ if (ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValues[k]) > 0) {
bitSet.set(i);
}
}
@@ -414,21 +416,24 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- super.readBlocks(blockChunkHolder);
+ super.readColumnChunks(rawBlockletColumnChunks);
}
- int blockIndex = dimensionBlocksIndex[0];
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = dimensionChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex = measureBlocksIndex[0];
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = measureChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index db55e42..3bb24af 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -32,11 +32,10 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -46,7 +45,7 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator;
public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilterExecuterImpl {
- protected byte[][] filterRangeValues;
+ private byte[][] filterRangeValues;
private Object[] msrFilterRangeValues;
private SerializableComparator comparator;
/**
@@ -55,7 +54,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
private boolean isDefaultValuePresentInFilter;
private int lastDimensionColOrdinal = 0;
- public RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
+ RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
@@ -115,11 +114,11 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
byte[] maxValue = null;
if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
- maxValue = blockMaxValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ maxValue = blockMaxValue[measureChunkIndex[0] + lastDimensionColOrdinal];
isScanRequired =
isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
- maxValue = blockMaxValue[dimensionBlocksIndex[0]];
+ maxValue = blockMaxValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(maxValue, filterRangeValues);
}
} else {
@@ -165,25 +164,26 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
@@ -195,26 +195,28 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
- } else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ } else {
+ int chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
@@ -224,28 +226,27 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
.getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i],
msrColEvalutorInfoList.get(0).getType()));
ColumnPage columnPage =
- rawColumnChunk.convertToColumnPage(i);
+ rawColumnChunk.decodeColumnPage(i);
if (compare <= 0 && columnPage.getNullBits().isEmpty()) {
BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
}
- return null;
}
private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage,
@@ -279,16 +280,16 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = null;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows);
} else {
- bitSet = setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ bitSet = setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows);
}
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet,
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet,
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
}
return bitSet;
@@ -300,12 +301,12 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
int last = 0;
@@ -313,7 +314,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
byte[][] filterValues = this.filterRangeValues;
for (int i = 0; i < filterValues.length; i++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i], false);
if (start < 0) {
start = -(start + 1);
@@ -324,14 +325,14 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
> 0) {
start = start + 1;
}
}
last = start;
for (int j = start; j < numerOfRows; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
last++;
}
startIndex = last;
@@ -348,11 +349,11 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
* be called if the column is sorted default so column index
* mapping will be present for accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
@@ -363,7 +364,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
int startIndex = 0;
for (int k = 0; k < filterValues.length; k++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, filterValues[k], false);
if (start < 0) {
start = -(start + 1);
@@ -373,7 +374,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) > 0) {
+ if (ByteUtil.compare(filterValues[k], dimensionColumnPage.getChunkData(start)) > 0) {
start = start + 1;
}
}
@@ -391,7 +392,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
} else {
for (int k = 0; k < filterValues.length; k++) {
for (int i = 0; i < numerOfRows; i++) {
- if (ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValues[k]) >= 0) {
+ if (ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValues[k]) >= 0) {
bitSet.set(i);
}
}
@@ -400,21 +401,24 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- super.readBlocks(blockChunkHolder);
+ super.readColumnChunks(rawBlockletColumnChunks);
}
- int blockIndex = dimensionBlocksIndex[0];
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = dimensionChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex = measureBlocksIndex[0];
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = measureChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/15b4e192/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 88cf75c..2e9bcb6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -35,11 +35,10 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -117,11 +116,11 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
boolean isScanRequired = false;
if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
- minValue = blockMinValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ minValue = blockMinValue[measureChunkIndex[0] + lastDimensionColOrdinal];
isScanRequired =
isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
- minValue = blockMinValue[dimensionBlocksIndex[0]];
+ minValue = blockMinValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(minValue, filterRangeValues);
}
} else {
@@ -167,60 +166,63 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], this.msrFilterRangeValues,
msrColEvalutorInfoList.get(0).getType())) {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -260,7 +262,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return bitSet;
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
byte[] defaultValue = null;
if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
@@ -269,7 +271,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
dimColEvaluatorInfoList.get(0).getDimension().getDataType());
int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
- segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
+ segmentProperties.getDimensions().get(dimensionChunkIndex[0]);
if (currentBlockDimension.isSortColumn()) {
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
this.segmentProperties.getSortColumnsGenerator());
@@ -280,14 +282,14 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
}
BitSet bitSet = null;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows,
+ if (dimensionColumnPage.isExplicitSorted()) {
+ bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows,
defaultValue);
} else {
- bitSet = setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows, defaultValue);
+ bitSet = setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows, defaultValue);
}
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet,
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet,
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
}
return bitSet;
@@ -299,12 +301,12 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows,
+ DimensionColumnPage dimensionColumnPage, int numerOfRows,
byte[] defaultValue) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
@@ -315,7 +317,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
//find the number of default values to skip the null value in case of direct dictionary
if (null != defaultValue) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
defaultValue, true);
if (start < 0) {
skip = -(start + 1);
@@ -332,7 +334,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
}
for (int i = 0; i < filterValues.length; i++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i], true);
if (start < 0) {
start = -(start + 1);
@@ -343,14 +345,14 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
// will be pointing to the next consecutive position. So compare it again and point to the
// previous value returned from getFirstIndexUsingBinarySearch.
if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
< 0) {
start = start - 1;
}
}
last = start;
for (int j = start; j >= skip; j--) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
last--;
}
startIndex = last;
@@ -367,12 +369,12 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
* be called if the column is sorted default so column index
* mapping will be present for accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @param defaultValue
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows, byte[] defaultValue) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
@@ -385,7 +387,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
//find the number of default values to skip the null value in case of direct dictionary
if (null != defaultValue) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, defaultValue, true);
if (start < 0) {
skip = -(start + 1);
@@ -402,7 +404,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
}
for (int k = 0; k < filterValues.length; k++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, filterValues[k], true);
if (start < 0) {
start = -(start + 1);
@@ -412,7 +414,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
// When negative value of start is returned from getFirstIndexUsingBinarySearch the Start
// will be pointing to the next consecutive position. So compare it again and point to the
// previous value returned from getFirstIndexUsingBinarySearch.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) < 0) {
+ if (ByteUtil.compare(filterValues[k], dimensionColumnPage.getChunkData(start)) < 0) {
start = start - 1;
}
}
@@ -429,7 +431,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
} else {
for (int k = 0; k < filterValues.length; k++) {
for (int i = 0; i < numerOfRows; i++) {
- if (ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValues[k]) <= 0) {
+ if (ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValues[k]) <= 0) {
bitSet.set(i);
}
}
@@ -438,21 +440,24 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- super.readBlocks(blockChunkHolder);
+ super.readColumnChunks(rawBlockletColumnChunks);
}
- int blockIndex = dimensionBlocksIndex[0];
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = dimensionChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex = measureBlocksIndex[0];
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = measureChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
[05/50] [abbrv] carbondata git commit: [CARBONDATA-2049]
CarbonCleanFilesCommand table path problem
Posted by ra...@apache.org.
[CARBONDATA-2049] CarbonCleanFilesCommand table path problem
Problem:
In CarbonCleanFilesCommand datbaseLocation is being passed instead of the tablePath in case of forceclean.
And in case of cleanGarbageData, storeLocation is being passed instead of the tablePath.
This closes #1828
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9b479617
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9b479617
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9b479617
Branch: refs/heads/fgdatamap
Commit: 9b4796177610e3a4f9d426169753a40eceb7b675
Parents: d509f17
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Tue Jan 16 11:49:54 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 09:37:39 2018 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/api/CarbonStore.scala | 32 ++++++++++++++------
.../management/CarbonCleanFilesCommand.scala | 25 +++++++--------
.../org/apache/spark/util/CleanFiles.scala | 32 ++++++++++++++------
.../apache/spark/util/CarbonCommandSuite.scala | 3 +-
4 files changed, 59 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b479617/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index d514f77..c02ba0a 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -98,34 +98,46 @@ object CarbonStore {
}
}
+ /**
+ * The method deletes all data if forceTableCLean <true> and lean garbage segment
+ * (MARKED_FOR_DELETE state) if forceTableCLean <false>
+ *
+ * @param dbName : Database name
+ * @param tableName : Table name
+ * @param tablePath : Table path
+ * @param carbonTable : CarbonTable Object <null> in case of force clean
+ * @param forceTableClean : <true> for force clean it will delete all data
+ * <false> it will clean garbage segment (MARKED_FOR_DELETE state)
+ * @param currentTablePartitions : Hive Partitions details
+ */
def cleanFiles(
dbName: String,
tableName: String,
- storePath: String,
+ tablePath: String,
carbonTable: CarbonTable,
forceTableClean: Boolean,
currentTablePartitions: Option[Seq[String]] = None): Unit = {
LOGGER.audit(s"The clean files request has been received for $dbName.$tableName")
var carbonCleanFilesLock: ICarbonLock = null
- var absoluteTableIdentifier: AbsoluteTableIdentifier = null
- if (forceTableClean) {
- absoluteTableIdentifier = AbsoluteTableIdentifier.from(storePath, dbName, tableName)
+ val absoluteTableIdentifier = if (forceTableClean) {
+ AbsoluteTableIdentifier.from(tablePath, dbName, tableName)
} else {
- absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
+ carbonTable.getAbsoluteTableIdentifier
}
try {
val errorMsg = "Clean files request is failed for " +
s"$dbName.$tableName" +
". Not able to acquire the clean files lock due to another clean files " +
"operation is running in the background."
- carbonCleanFilesLock =
- CarbonLockUtil.getLockObject(absoluteTableIdentifier, LockUsage.CLEAN_FILES_LOCK, errorMsg)
+ // in case of force clean the lock is not required
if (forceTableClean) {
- val absIdent = AbsoluteTableIdentifier.from(storePath, dbName, tableName)
FileFactory.deleteAllCarbonFilesOfDir(
- FileFactory.getCarbonFile(absIdent.getTablePath,
- FileFactory.getFileType(absIdent.getTablePath)))
+ FileFactory.getCarbonFile(absoluteTableIdentifier.getTablePath,
+ FileFactory.getFileType(absoluteTableIdentifier.getTablePath)))
} else {
+ carbonCleanFilesLock =
+ CarbonLockUtil
+ .getLockObject(absoluteTableIdentifier, LockUsage.CLEAN_FILES_LOCK, errorMsg)
DataLoadingUtil.deleteLoadsAndUpdateMetadata(
isForceDeletion = true, carbonTable)
CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, true)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b479617/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
index 303c3ef..4b68bd0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.events.{CleanFilesPostEvent, CleanFilesPreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.spark.util.CommonUtil
@@ -70,12 +70,13 @@ case class CarbonCleanFilesCommand(
databaseNameOp: Option[String], tableName: String): Unit = {
val dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
val databaseLocation = CarbonEnv.getDatabaseLocation(dbName, sparkSession)
+ val tablePath = databaseLocation + CarbonCommonConstants.FILE_SEPARATOR + tableName
CarbonStore.cleanFiles(
- dbName,
- tableName,
- databaseLocation,
- null,
- forceTableClean)
+ dbName = dbName,
+ tableName = tableName,
+ tablePath = tablePath,
+ carbonTable = null, // in case of delete all data carbonTable is not required.
+ forceTableClean = forceTableClean)
}
private def cleanGarbageData(sparkSession: SparkSession,
@@ -90,12 +91,12 @@ case class CarbonCleanFilesCommand(
None
}
CarbonStore.cleanFiles(
- CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession),
- tableName,
- CarbonProperties.getStorePath,
- carbonTable,
- forceTableClean,
- partitions)
+ dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession),
+ tableName = tableName,
+ tablePath = carbonTable.getTablePath,
+ carbonTable = carbonTable,
+ forceTableClean = forceTableClean,
+ currentTablePartitions = partitions)
}
// Clean garbage data in all tables in all databases
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b479617/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
index eba7dcd..d4d9a84 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
@@ -29,19 +29,30 @@ import org.apache.carbondata.api.CarbonStore
object CleanFiles {
/**
- * Clean the stale segments from table
- * @param spark
- * @param dbName
- * @param tableName
- * @param storePath
- * @param forceTableClean if true, it deletes the table and its contents with force.It does not
+ * The method deletes all data if forceTableCLean <true> and lean garbage segment
+ * (MARKED_FOR_DELETE state) if forceTableCLean <false>
+ *
+ * @param spark : Database name
+ * @param dbName : Table name
+ * @param tableName : Table path
+ * @param forceTableClean : if true, it deletes the table and its contents with force.It does not
* drop table from hive metastore so should be very careful to use it.
*/
def cleanFiles(spark: SparkSession, dbName: String, tableName: String,
- storePath: String, forceTableClean: Boolean = false): Unit = {
+ forceTableClean: Boolean = false): Unit = {
TableAPIUtil.validateTableExists(spark, dbName, tableName)
- val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(spark)
- CarbonStore.cleanFiles(dbName, tableName, storePath, carbonTable, forceTableClean)
+ val tablePath = CarbonEnv.getTablePath(Some(dbName), tableName)(spark)
+ val carbonTable = if (!forceTableClean) {
+ CarbonEnv.getCarbonTable(Some(dbName), tableName)(spark)
+ } else {
+ null
+ }
+ CarbonStore.cleanFiles(
+ dbName = dbName,
+ tableName = tableName,
+ tablePath = tablePath,
+ carbonTable = carbonTable,
+ forceTableClean = forceTableClean)
}
def main(args: Array[String]): Unit = {
@@ -60,6 +71,7 @@ object CleanFiles {
val spark = TableAPIUtil.spark(storePath, s"CleanFiles: $dbName.$tableName")
CarbonEnv.getInstance(spark).carbonMetastore.
checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
- cleanFiles(spark, dbName, tableName, storePath, forceTableClean)
+
+ cleanFiles(spark, dbName, tableName, forceTableClean)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b479617/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
index e493179..8ff6cab 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
@@ -142,7 +142,8 @@ class CarbonCommandSuite extends Spark2QueryTest with BeforeAndAfterAll {
dropTable(table)
createAndLoadTestTable(table, "csv_table")
CleanFiles.main(Array(s"${location}", table, "true"))
- val tablePath = s"${location}${File.separator}default${File.separator}$table"
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", table)
+ val tablePath = carbonTable.getTablePath
val f = new File(tablePath)
assert(!f.exists())
[11/50] [abbrv] carbondata git commit: [CARBONDATA-1948] Update help
document for the change made for CARBONDATA-1929
Posted by ra...@apache.org.
[CARBONDATA-1948] Update help document for the change made for CARBONDATA-1929
Update help document for the change made for CARBONDATA-1929
This closes #1884
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8314ea21
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8314ea21
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8314ea21
Branch: refs/heads/fgdatamap
Commit: 8314ea21a2c9bed59e86864dda055be8be6011e3
Parents: c9a501d
Author: Varun Saxena <va...@apache.org>
Authored: Tue Jan 30 16:30:36 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jan 31 12:05:10 2018 +0800
----------------------------------------------------------------------
docs/configuration-parameters.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8314ea21/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 5794f4d..367d045 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -96,9 +96,9 @@ This section provides the details of all the configurations required for CarbonD
| Parameter | Default Value | Description |
|---------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| carbon.sort.file.write.buffer.size | 10485760 | File write buffer size used during sorting. |
+| carbon.sort.file.write.buffer.size | 16777216 | File write buffer size used during sorting (minValue = 10 KB, maxValue=10MB). |
| carbon.lock.type | LOCALLOCK | This configuration specifies the type of lock to be acquired during concurrent operations on table. There are following types of lock implementation: - LOCALLOCK: Lock is created on local file system as file. This lock is useful when only one spark driver (thrift server) runs on a machine and no other CarbonData spark application is launched concurrently. - HDFSLOCK: Lock is created on HDFS file system as file. This lock is useful when multiple CarbonData spark applications are launched and no ZooKeeper is running on cluster and HDFS supports file based locking. |
-| carbon.sort.intermediate.files.limit | 20 | Minimum number of intermediate files after which merged sort can be started. |
+| carbon.sort.intermediate.files.limit | 20 | Minimum number of intermediate files after which merged sort can be started (minValue = 2, maxValue=50). |
| carbon.block.meta.size.reserved.percentage | 10 | Space reserved in percentage for writing block meta data in CarbonData file. |
| carbon.csv.read.buffersize.byte | 1048576 | csv reading buffer size. |
| high.cardinality.value | 100000 | To identify and apply compression for non-high cardinality columns. |
[29/50] [abbrv] carbondata git commit: [REBASE] Solve conflict after
rebasing master
Posted by ra...@apache.org.
[REBASE] Solve conflict after rebasing master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/71c2d8ca
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/71c2d8ca
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/71c2d8ca
Branch: refs/heads/fgdatamap
Commit: 71c2d8ca4a3212cff1eedbe78ee03e521f57fbbc
Parents: 80b42ac
Author: Jacky Li <ja...@qq.com>
Authored: Thu Feb 1 00:25:31 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Feb 1 00:25:31 2018 +0800
----------------------------------------------------------------------
.../hadoop/util/CarbonInputFormatUtil.java | 20 +++++++++++++++++++
.../spark/rdd/NewCarbonDataLoadRDD.scala | 21 ++------------------
.../org/apache/spark/sql/CarbonSession.scala | 5 ++---
3 files changed, 24 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/71c2d8ca/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 514428b..056c27b 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -22,6 +22,8 @@ import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Locale;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -39,6 +41,7 @@ import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
@@ -159,4 +162,21 @@ public class CarbonInputFormatUtil {
String jobtrackerID = createJobTrackerID(date);
return new JobID(jobtrackerID, batch);
}
+
+ public static void setS3Configurations(Configuration hadoopConf) {
+ FileFactory.getConfiguration()
+ .set("fs.s3a.access.key", hadoopConf.get("fs.s3a.access.key", ""));
+ FileFactory.getConfiguration()
+ .set("fs.s3a.secret.key", hadoopConf.get("fs.s3a.secret.key", ""));
+ FileFactory.getConfiguration()
+ .set("fs.s3a.endpoint", hadoopConf.get("fs.s3a.endpoint", ""));
+ FileFactory.getConfiguration().set(CarbonCommonConstants.S3_ACCESS_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3_ACCESS_KEY, ""));
+ FileFactory.getConfiguration().set(CarbonCommonConstants.S3_SECRET_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3_SECRET_KEY, ""));
+ FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_ACCESS_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3N_ACCESS_KEY, ""));
+ FileFactory.getConfiguration().set(CarbonCommonConstants.S3N_SECRET_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3N_SECRET_KEY, ""));
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/71c2d8ca/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 917fc88..e17824f 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -41,10 +41,10 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.compression.CompressorFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.processing.loading.{DataLoadExecutor, FailureCauses, TableProcessingOperations}
import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat, CSVRecordReaderIterator}
import org.apache.carbondata.processing.loading.exception.NoRetryException
@@ -371,7 +371,7 @@ class NewDataFrameLoaderRDD[K, V](
override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val hadoopConf = getConf
- setS3Configurations(hadoopConf)
+ CarbonInputFormatUtil.setS3Configurations(hadoopConf)
val iter = new Iterator[(K, V)] {
val loadMetadataDetails = new LoadMetadataDetails()
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
@@ -441,23 +441,6 @@ class NewDataFrameLoaderRDD[K, V](
iter
}
override protected def getPartitions: Array[Partition] = firstParent[Row].partitions
-
- private def setS3Configurations(hadoopConf: Configuration): Unit = {
- FileFactory.getConfiguration
- .set("fs.s3a.access.key", hadoopConf.get("fs.s3a.access.key", ""))
- FileFactory.getConfiguration
- .set("fs.s3a.secret.key", hadoopConf.get("fs.s3a.secret.key", ""))
- FileFactory.getConfiguration
- .set("fs.s3a.endpoint", hadoopConf.get("fs.s3a.endpoint", ""))
- FileFactory.getConfiguration.set(CarbonCommonConstants.S3_ACCESS_KEY,
- hadoopConf.get(CarbonCommonConstants.S3_ACCESS_KEY, ""))
- FileFactory.getConfiguration.set(CarbonCommonConstants.S3_SECRET_KEY,
- hadoopConf.get(CarbonCommonConstants.S3_SECRET_KEY, ""))
- FileFactory.getConfiguration.set(CarbonCommonConstants.S3N_ACCESS_KEY,
- hadoopConf.get(CarbonCommonConstants.S3N_ACCESS_KEY, ""))
- FileFactory.getConfiguration.set(CarbonCommonConstants.S3N_SECRET_KEY,
- hadoopConf.get(CarbonCommonConstants.S3N_SECRET_KEY, ""))
- }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/71c2d8ca/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index ded8f35..28471f0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -21,7 +21,6 @@ import java.io.File
import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.s3a.Constants.{ACCESS_KEY, ENDPOINT, SECRET_KEY}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
import org.apache.spark.sql.SparkSession.Builder
@@ -31,8 +30,8 @@ import org.apache.spark.sql.internal.{SessionState, SharedState}
import org.apache.spark.util.{CarbonReflectionUtils, Utils}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, ThreadLocalSessionInfo}
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
/**
* Session implementation for {org.apache.spark.sql.SparkSession}
@@ -154,7 +153,7 @@ object CarbonSession {
sparkConf.setAppName(randomAppName)
}
val sc = SparkContext.getOrCreate(sparkConf)
- setS3Configurations(sc)
+ CarbonInputFormatUtil.setS3Configurations(sc.hadoopConfiguration)
// maybe this is an existing SparkContext, update its SparkConf which maybe used
// by SparkSession
options.foreach { case (k, v) => sc.conf.set(k, v) }
[28/50] [abbrv] carbondata git commit: [CARBONDATA-1827] S3 Carbon
Implementation
Posted by ra...@apache.org.
[CARBONDATA-1827] S3 Carbon Implementation
1.Provide support for s3 in carbondata.
2.Added S3Example to create carbon table on s3.
3.Added S3CSVExample to load carbon table using csv from s3.
This closes #1805
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/80b42ac6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/80b42ac6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/80b42ac6
Branch: refs/heads/fgdatamap
Commit: 80b42ac662ebd2bc243ca91c86b035717223daf4
Parents: 111c382
Author: SangeetaGulia <sa...@knoldus.in>
Authored: Thu Sep 21 14:56:26 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Feb 1 00:16:40 2018 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 21 +++
.../filesystem/AbstractDFSCarbonFile.java | 20 ++-
.../datastore/filesystem/HDFSCarbonFile.java | 5 +-
.../core/datastore/impl/FileFactory.java | 11 +-
.../core/locks/CarbonLockFactory.java | 28 ++--
.../carbondata/core/locks/S3FileLock.java | 111 +++++++++++++
.../carbondata/core/util/CarbonProperties.java | 3 +-
.../filesystem/HDFSCarbonFileTest.java | 8 +-
examples/spark2/pom.xml | 5 +
examples/spark2/src/main/resources/data1.csv | 11 ++
.../carbondata/examples/S3CsvExample.scala | 99 +++++++++++
.../apache/carbondata/examples/S3Example.scala | 164 +++++++++++++++++++
.../spark/rdd/NewCarbonDataLoadRDD.scala | 42 ++++-
integration/spark2/pom.xml | 43 +++++
.../spark/rdd/CarbonDataRDDFactory.scala | 3 +-
.../org/apache/spark/sql/CarbonSession.scala | 3 +
16 files changed, 554 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 77e8db8..4d838a8 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -167,6 +167,22 @@ public final class CarbonCommonConstants {
public static final String S3N_PREFIX = "s3n://";
public static final String S3A_PREFIX = "s3a://";
+ /**
+ * Access Key for s3n
+ */
+ public static final String S3N_ACCESS_KEY = "fs.s3n.awsAccessKeyId";
+ /**
+ * Secret Key for s3n
+ */
+ public static final String S3N_SECRET_KEY = "fs.s3n.awsSecretAccessKey";
+ /**
+ * Access Key for s3
+ */
+ public static final String S3_ACCESS_KEY = "fs.s3.awsAccessKeyId";
+ /**
+ * Secret Key for s3
+ */
+ public static final String S3_SECRET_KEY = "fs.s3.awsSecretAccessKey";
/**
* FS_DEFAULT_FS
@@ -941,6 +957,11 @@ public final class CarbonCommonConstants {
public static final String CARBON_LOCK_TYPE_HDFS = "HDFSLOCK";
/**
+ * S3LOCK TYPE
+ */
+ public static final String CARBON_LOCK_TYPE_S3 = "S3LOCK";
+
+ /**
* Invalid filter member log string
*/
public static final String FILTER_INVALID_MEMBER =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index 7b634d2..e1a34fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
-public abstract class AbstractDFSCarbonFile implements CarbonFile {
+public abstract class AbstractDFSCarbonFile implements CarbonFile {
/**
* LOGGER
*/
@@ -261,18 +261,28 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
@Override public DataOutputStream getDataOutputStream(String path, FileFactory.FileType fileType,
int bufferSize, boolean append) throws IOException {
Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
+ FileSystem fileSystem = pt.getFileSystem(FileFactory.getConfiguration());
FSDataOutputStream stream = null;
if (append) {
// append to a file only if file already exists else file not found
// exception will be thrown by hdfs
if (CarbonUtil.isFileExists(path)) {
- stream = fs.append(pt, bufferSize);
+ if (FileFactory.FileType.S3 == fileType) {
+ DataInputStream dataInputStream = fileSystem.open(pt);
+ int count = dataInputStream.available();
+ // create buffer
+ byte[] byteStreamBuffer = new byte[count];
+ dataInputStream.read(byteStreamBuffer);
+ stream = fileSystem.create(pt, true, bufferSize);
+ stream.write(byteStreamBuffer);
+ } else {
+ stream = fileSystem.append(pt, bufferSize);
+ }
} else {
- stream = fs.create(pt, true, bufferSize);
+ stream = fileSystem.create(pt, true, bufferSize);
}
} else {
- stream = fs.create(pt, true, bufferSize);
+ stream = fileSystem.create(pt, true, bufferSize);
}
return stream;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
index d470b47..892a556 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
@@ -107,8 +107,11 @@ public class HDFSCarbonFile extends AbstractDFSCarbonFile {
((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
org.apache.hadoop.fs.Options.Rename.OVERWRITE);
return true;
+ } else if (fileStatus.getPath().toString().startsWith("s3n")) {
+ fs.delete(new Path(changetoName), true);
+ return fs.rename(fileStatus.getPath(), new Path(changetoName));
} else {
- return false;
+ return fs.rename(fileStatus.getPath(), new Path(changetoName));
}
} catch (IOException e) {
LOGGER.error("Exception occured: " + e.getMessage());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index e6fbd04..daf6d93 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -246,7 +246,15 @@ public final class FileFactory {
*/
public static DataOutputStream getDataOutputStreamUsingAppend(String path, FileType fileType)
throws IOException {
- return getCarbonFile(path).getDataOutputStreamUsingAppend(path, fileType);
+ if (FileType.S3 == fileType) {
+ CarbonFile carbonFile = getCarbonFile(path);
+ if (carbonFile.exists()) {
+ carbonFile.delete();
+ }
+ return carbonFile.getDataOutputStream(path,fileType);
+ } else {
+ return getCarbonFile(path).getDataOutputStreamUsingAppend(path, fileType);
+ }
}
/**
@@ -423,6 +431,7 @@ public final class FileFactory {
throws IOException {
FileFactory.FileType fileType = FileFactory.getFileType(directoryPath);
switch (fileType) {
+ case S3:
case HDFS:
case VIEWFS:
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
index e70e655..3226a63 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
@@ -52,18 +52,21 @@ public class CarbonLockFactory {
*/
public static ICarbonLock getCarbonLockObj(AbsoluteTableIdentifier absoluteTableIdentifier,
String lockFile) {
- switch (lockTypeConfigured) {
- case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
- return new LocalFileLock(absoluteTableIdentifier, lockFile);
- case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
- return new ZooKeeperLocking(absoluteTableIdentifier, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
- return new HdfsFileLock(absoluteTableIdentifier, lockFile);
-
- default:
- throw new UnsupportedOperationException("Not supported the lock type");
+ String tablePath = absoluteTableIdentifier.getTablePath();
+ if (lockTypeConfigured.equals(CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER)) {
+ return new ZooKeeperLocking(absoluteTableIdentifier, lockFile);
+ } else if (tablePath.startsWith(CarbonCommonConstants.S3A_PREFIX) ||
+ tablePath.startsWith(CarbonCommonConstants.S3N_PREFIX) ||
+ tablePath.startsWith(CarbonCommonConstants.S3_PREFIX)) {
+ lockTypeConfigured = CarbonCommonConstants.CARBON_LOCK_TYPE_S3;
+ return new S3FileLock(absoluteTableIdentifier, lockFile);
+ } else if (tablePath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) {
+ lockTypeConfigured = CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS;
+ return new HdfsFileLock(absoluteTableIdentifier, lockFile);
+ } else {
+ lockTypeConfigured = CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL;
+ return new LocalFileLock(absoluteTableIdentifier, lockFile);
}
}
@@ -84,6 +87,9 @@ public class CarbonLockFactory {
case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
return new HdfsFileLock(locFileLocation, lockFile);
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_S3:
+ return new S3FileLock(locFileLocation, lockFile);
+
default:
throw new UnsupportedOperationException("Not supported the lock type");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/core/src/main/java/org/apache/carbondata/core/locks/S3FileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/S3FileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/S3FileLock.java
new file mode 100644
index 0000000..8836960
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/S3FileLock.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.locks;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+
+/**
+ * This class is used to handle the S3 File locking.
+ * This is acheived using the concept of acquiring the data out stream using Append option.
+ */
+public class S3FileLock extends AbstractCarbonLock {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(S3FileLock.class.getName());
+ /**
+ * location s3 file location
+ */
+ private String location;
+
+ private DataOutputStream dataOutputStream;
+
+ /**
+ * @param tableIdentifier
+ * @param lockFile
+ */
+ public S3FileLock(AbsoluteTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getTablePath(), lockFile);
+ }
+
+ /**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public S3FileLock(String lockFileLocation, String lockFile) {
+ this.location = lockFileLocation + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ LOGGER.info("S3 lock path:" + this.location);
+ initRetry();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.carbondata.core.locks.ICarbonLock#unlock()
+ */
+ @Override public boolean unlock() {
+ boolean status = false;
+ if (null != dataOutputStream) {
+ try {
+ dataOutputStream.close();
+ status = true;
+ } catch (IOException e) {
+ status = false;
+ } finally {
+ CarbonFile carbonFile =
+ FileFactory.getCarbonFile(location, FileFactory.getFileType(location));
+ if (carbonFile.exists()) {
+ if (carbonFile.delete()) {
+ LOGGER.info("Deleted the lock file " + location);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + location);
+ status = false;
+ }
+ } else {
+ LOGGER.error(
+ "Not able to delete the lock file because it is not existed in location " + location);
+ status = false;
+ }
+ }
+ }
+ return status;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.carbondata.core.locks.ICarbonLock#lock()
+ */
+ @Override public boolean lock() {
+ try {
+ if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
+ FileFactory.createNewLockFile(location, FileFactory.getFileType(location));
+ }
+ dataOutputStream =
+ FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location));
+ return true;
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return false;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index fd78efc..13c140f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -306,7 +306,8 @@ public final class CarbonProperties {
String defaultFs = configuration.get("fs.defaultFS");
if (null != defaultFs && (defaultFs.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)
|| defaultFs.startsWith(CarbonCommonConstants.VIEWFSURL_PREFIX) || defaultFs
- .startsWith(CarbonCommonConstants.ALLUXIOURL_PREFIX))
+ .startsWith(CarbonCommonConstants.ALLUXIOURL_PREFIX) || defaultFs
+ .startsWith(CarbonCommonConstants.S3A_PREFIX))
&& !CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS.equalsIgnoreCase(lockTypeConfigured)) {
LOGGER.warn("The value \"" + lockTypeConfigured + "\" configured for key "
+ LOCK_TYPE + " is invalid for current file system. "
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
index 7726693..4018123 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -369,7 +369,13 @@ public class HDFSCarbonFileTest {
}
};
- assertEquals(hdfsCarbonFile.renameForce(fileName), false);
+ new MockUp<WebHdfsFileSystem>(){
+ @Mock
+ public boolean rename(final Path src, final Path dst) throws IOException {
+ return true;
+ }
+ };
+ assertEquals(hdfsCarbonFile.renameForce(fileName), true);
}
@Test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/examples/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index da39f1d..62679be 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -62,6 +62,11 @@
<version>${spark.version}</version>
<scope>${spark.deps.scope}</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.2</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/examples/spark2/src/main/resources/data1.csv
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/resources/data1.csv b/examples/spark2/src/main/resources/data1.csv
new file mode 100644
index 0000000..cf732eb
--- /dev/null
+++ b/examples/spark2/src/main/resources/data1.csv
@@ -0,0 +1,11 @@
+shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField
+1,10,1100,48.4,spark,2015-4-23 12:01:01,1.23,2015-4-23,aaa,2.5
+5,17,1140,43.4,spark,2015-7-27 12:01:02,3.45,2015-7-27,bbb,2.5
+1,11,1100,44.4,flink,2015-5-23 12:01:03,23.23,2015-5-23,ccc,2.5
+1,10,1150,43.4,spark,2015-7-24 12:01:04,254.12,2015-7-24,ddd,2.5
+1,10,1100,47.4,spark,2015-7-23 12:01:05,876.14,2015-7-23,eeee,3.5
+3,14,1160,43.4,hive,2015-7-26 12:01:06,3454.32,2015-7-26,ff,2.5
+2,10,1100,43.4,impala,2015-7-23 12:01:07,456.98,2015-7-23,ggg,2.5
+1,10,1100,43.4,spark,2015-5-23 12:01:08,32.53,2015-5-23,hhh,2.5
+4,16,1130,42.4,impala,2015-7-23 12:01:09,67.23,2015-7-23,iii,2.5
+1,10,1100,43.4,spark,2015-7-23 12:01:10,832.23,2015-7-23,jjj,2.5
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3CsvExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3CsvExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3CsvExample.scala
new file mode 100644
index 0000000..b37fba8
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3CsvExample.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.hadoop.fs.s3a.Constants.{ACCESS_KEY, SECRET_KEY}
+import org.apache.spark.sql.SparkSession
+import org.slf4j.{Logger, LoggerFactory}
+
+object S3CsvExample {
+
+ /**
+ * This example demonstrate to create local store and load data from CSV files on S3
+ *
+ * @param args require three parameters "Access-key" "Secret-key"
+ * "s3 path to csv" "spark-master"
+ */
+ def main(args: Array[String]) {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val logger: Logger = LoggerFactory.getLogger(this.getClass)
+
+ import org.apache.spark.sql.CarbonSession._
+ if (args.length != 4) {
+ logger.error("Usage: java CarbonS3Example <access-key> <secret-key>" +
+ "<s3.csv.location> <spark-master>")
+ System.exit(0)
+ }
+
+ val spark = SparkSession
+ .builder()
+ .master(args(3))
+ .appName("S3CsvExample")
+ .config("spark.driver.host", "localhost")
+ .config("spark.hadoop." + ACCESS_KEY, args(0))
+ .config("spark.hadoop." + SECRET_KEY, args(1))
+ .getOrCreateCarbonSession()
+
+ spark.sparkContext.setLogLevel("INFO")
+
+ spark.sql(
+ s"""
+ | CREATE TABLE if not exists carbon_table1(
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT
+ | )
+ | STORED BY 'carbondata'
+ | LOCATION '$rootPath/examples/spark2/target/store'
+ | TBLPROPERTIES('SORT_COLUMNS'='', 'DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${ args(2) }'
+ | INTO TABLE carbon_table1
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${ args(2) }'
+ | INTO TABLE carbon_table1
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table1
+ """.stripMargin).show()
+
+ spark.sql("Drop table if exists carbon_table1")
+
+ spark.stop()
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
new file mode 100644
index 0000000..d3d0a37
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.hadoop.fs.s3a.Constants.{ACCESS_KEY, ENDPOINT, SECRET_KEY}
+import org.apache.spark.sql.{Row, SparkSession}
+import org.slf4j.{Logger, LoggerFactory}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+
+object S3Example {
+
+ /**
+ * This example demonstrate usage of
+ * 1. create carbon table with storage location on object based storage
+ * like AWS S3, Huawei OBS, etc
+ * 2. load data into carbon table, the generated file will be stored on object based storage
+ * query the table.
+ *
+ * @param args require three parameters "Access-key" "Secret-key"
+ * "table-path on s3" "s3-endpoint" "spark-master"
+ */
+ def main(args: Array[String]) {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val path = s"$rootPath/examples/spark2/src/main/resources/data1.csv"
+ val logger: Logger = LoggerFactory.getLogger(this.getClass)
+
+ import org.apache.spark.sql.CarbonSession._
+ if (args.length < 3 || args.length > 5) {
+ logger.error("Usage: java CarbonS3Example <access-key> <secret-key>" +
+ "<table-path-on-s3> [s3-endpoint] [spark-master]")
+ System.exit(0)
+ }
+
+ val (accessKey, secretKey, endpoint) = getKeyOnPrefix(args(2))
+ val spark = SparkSession
+ .builder()
+ .master(getSparkMaster(args))
+ .appName("S3Example")
+ .config("spark.driver.host", "localhost")
+ .config(accessKey, args(0))
+ .config(secretKey, args(1))
+ .config(endpoint, getS3EndPoint(args))
+ .getOrCreateCarbonSession()
+
+ spark.sparkContext.setLogLevel("WARN")
+
+ spark.sql("Drop table if exists carbon_table")
+
+ spark.sql(
+ s"""
+ | CREATE TABLE if not exists carbon_table(
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT
+ | )
+ | STORED BY 'carbondata'
+ | LOCATION '${ args(2) }'
+ | TBLPROPERTIES('SORT_COLUMNS'='', 'DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon_table
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ """.stripMargin).show()
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon_table
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon_table
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ val countSegment: Array[Row] =
+ spark.sql(
+ s"""
+ | SHOW SEGMENTS FOR TABLE carbon_table
+ """.stripMargin).collect()
+
+ while (countSegment.length != 3) {
+ this.wait(2000)
+ }
+
+ // Use compaction command to merge segments or small files in object based storage,
+ // this can be done periodically.
+ spark.sql("ALTER table carbon_table compact 'MAJOR'")
+ spark.sql("show segments for table carbon_table").show()
+
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ """.stripMargin).show()
+
+ spark.sql("Drop table if exists carbon_table")
+
+ spark.stop()
+ }
+
+ def getKeyOnPrefix(path: String): (String, String, String) = {
+ val endPoint = "spark.hadoop." + ENDPOINT
+ if (path.startsWith(CarbonCommonConstants.S3A_PREFIX)) {
+ ("spark.hadoop." + ACCESS_KEY, "spark.hadoop." + SECRET_KEY, endPoint)
+ } else if (path.startsWith(CarbonCommonConstants.S3N_PREFIX)) {
+ ("spark.hadoop." + CarbonCommonConstants.S3N_ACCESS_KEY,
+ "spark.hadoop." + CarbonCommonConstants.S3N_SECRET_KEY, endPoint)
+ } else if (path.startsWith(CarbonCommonConstants.S3_PREFIX)) {
+ ("spark.hadoop." + CarbonCommonConstants.S3_ACCESS_KEY,
+ "spark.hadoop." + CarbonCommonConstants.S3_SECRET_KEY, endPoint)
+ } else {
+ throw new Exception("Incorrect Store Path")
+ }
+ }
+
+ def getS3EndPoint(args: Array[String]): String = {
+ if (args.length >= 4 && args(3).contains(".com")) args(3)
+ else ""
+ }
+
+ def getSparkMaster(args: Array[String]): String = {
+ if (args.length == 5) args(4)
+ else if (args(3).contains("spark:") || args(3).contains("mesos:")) args(3)
+ else "local"
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 1fa1689..917fc88 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -41,6 +41,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.compression.CompressorFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -346,11 +347,31 @@ class NewDataFrameLoaderRDD[K, V](
sc: SparkContext,
result: DataLoadResult[K, V],
carbonLoadModel: CarbonLoadModel,
- prev: DataLoadCoalescedRDD[Row]) extends CarbonRDD[(K, V)](prev) {
+ prev: DataLoadCoalescedRDD[Row],
+ @transient hadoopConf: Configuration) extends CarbonRDD[(K, V)](prev) {
- override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
+ private val confBytes = {
+ val bao = new ByteArrayOutputStream()
+ val oos = new ObjectOutputStream(bao)
+ hadoopConf.write(oos)
+ oos.close()
+ CompressorFactory.getInstance().getCompressor.compressByte(bao.toByteArray)
+ }
+ private def getConf = {
+ val configuration = new Configuration(false)
+ val bai = new ByteArrayInputStream(CompressorFactory.getInstance().getCompressor
+ .unCompressByte(confBytes))
+ val ois = new ObjectInputStream(bai)
+ configuration.readFields(ois)
+ ois.close()
+ configuration
+ }
+
+ override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+ val hadoopConf = getConf
+ setS3Configurations(hadoopConf)
val iter = new Iterator[(K, V)] {
val loadMetadataDetails = new LoadMetadataDetails()
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
@@ -420,6 +441,23 @@ class NewDataFrameLoaderRDD[K, V](
iter
}
override protected def getPartitions: Array[Partition] = firstParent[Row].partitions
+
+ private def setS3Configurations(hadoopConf: Configuration): Unit = {
+ FileFactory.getConfiguration
+ .set("fs.s3a.access.key", hadoopConf.get("fs.s3a.access.key", ""))
+ FileFactory.getConfiguration
+ .set("fs.s3a.secret.key", hadoopConf.get("fs.s3a.secret.key", ""))
+ FileFactory.getConfiguration
+ .set("fs.s3a.endpoint", hadoopConf.get("fs.s3a.endpoint", ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3_ACCESS_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3_ACCESS_KEY, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3_SECRET_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3_SECRET_KEY, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3N_ACCESS_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3N_ACCESS_KEY, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3N_SECRET_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3N_SECRET_KEY, ""))
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index 60cb61f..4a37abc 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -57,6 +57,49 @@
<version>2.2.1</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.7.4</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ <version>0.9.0</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 6f056a6..655e5a0 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -992,7 +992,8 @@ object CarbonDataRDDFactory {
sqlContext.sparkContext,
new DataLoadResultImpl(),
carbonLoadModel,
- newRdd
+ newRdd,
+ sqlContext.sparkContext.hadoopConfiguration
).collect()
} catch {
case ex: Exception =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/80b42ac6/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index e95b8db..ded8f35 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -21,6 +21,7 @@ import java.io.File
import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.s3a.Constants.{ACCESS_KEY, ENDPOINT, SECRET_KEY}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
import org.apache.spark.sql.SparkSession.Builder
@@ -30,6 +31,7 @@ import org.apache.spark.sql.internal.{SessionState, SharedState}
import org.apache.spark.util.{CarbonReflectionUtils, Utils}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, ThreadLocalSessionInfo}
/**
@@ -152,6 +154,7 @@ object CarbonSession {
sparkConf.setAppName(randomAppName)
}
val sc = SparkContext.getOrCreate(sparkConf)
+ setS3Configurations(sc)
// maybe this is an existing SparkContext, update its SparkConf which maybe used
// by SparkSession
options.foreach { case (k, v) => sc.conf.set(k, v) }
[21/50] [abbrv] carbondata git commit: [CARBONDATA-2021]fix clean up
issue when update operation is abprutly stopped
Posted by ra...@apache.org.
[CARBONDATA-2021]fix clean up issue when update operation is abprutly stopped
when delete is success and update is failed while writing status file then a stale carbon data file is created.
so removing that file on clean up . and also not considering that one during query.
This closes #1793
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b2139cab
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b2139cab
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b2139cab
Branch: refs/heads/fgdatamap
Commit: b2139cabe8cdeb7c241e30a525d754578cfa5ec6
Parents: d90280a
Author: akashrn5 <ak...@gmail.com>
Authored: Wed Jan 10 20:29:43 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 19:23:55 2018 +0800
----------------------------------------------------------------------
.../core/mutate/CarbonUpdateUtil.java | 64 ++++++++++++++++++--
.../SegmentUpdateStatusManager.java | 27 +++++++--
.../apache/carbondata/core/util/CarbonUtil.java | 10 +++
.../processing/util/CarbonLoaderUtil.java | 6 +-
4 files changed, 93 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b2139cab/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index f4566ac..0e4eec7 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -427,6 +427,10 @@ public class CarbonUpdateUtil {
String validUpdateStatusFile = "";
+ boolean isAbortedFile = true;
+
+ boolean isInvalidFile = false;
+
// scan through each segment.
for (LoadMetadataDetails segment : details) {
@@ -450,10 +454,14 @@ public class CarbonUpdateUtil {
SegmentUpdateStatusManager updateStatusManager =
new SegmentUpdateStatusManager(table.getAbsoluteTableIdentifier());
+ // deleting of the aborted file scenario.
+ deleteStaleCarbonDataFiles(segment, allSegmentFiles, updateStatusManager);
+
// get Invalid update delta files.
CarbonFile[] invalidUpdateDeltaFiles = updateStatusManager
- .getUpdateDeltaFilesList(segment.getLoadName(), false,
- CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, true, allSegmentFiles);
+ .getUpdateDeltaFilesList(segment.getLoadName(), false,
+ CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, true, allSegmentFiles,
+ isInvalidFile);
// now for each invalid delta file need to check the query execution time out
// and then delete.
@@ -465,8 +473,9 @@ public class CarbonUpdateUtil {
// do the same for the index files.
CarbonFile[] invalidIndexFiles = updateStatusManager
- .getUpdateDeltaFilesList(segment.getLoadName(), false,
- CarbonCommonConstants.UPDATE_INDEX_FILE_EXT, true, allSegmentFiles);
+ .getUpdateDeltaFilesList(segment.getLoadName(), false,
+ CarbonCommonConstants.UPDATE_INDEX_FILE_EXT, true, allSegmentFiles,
+ isInvalidFile);
// now for each invalid index file need to check the query execution time out
// and then delete.
@@ -492,11 +501,20 @@ public class CarbonUpdateUtil {
continue;
}
+ // aborted scenario.
+ invalidDeleteDeltaFiles = updateStatusManager
+ .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, false,
+ allSegmentFiles, isAbortedFile);
+ for (CarbonFile invalidFile : invalidDeleteDeltaFiles) {
+ boolean doForceDelete = true;
+ compareTimestampsAndDelete(invalidFile, doForceDelete, false);
+ }
+
// case 1
if (CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) {
completeListOfDeleteDeltaFiles = updateStatusManager
.getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, true,
- allSegmentFiles);
+ allSegmentFiles, isInvalidFile);
for (CarbonFile invalidFile : completeListOfDeleteDeltaFiles) {
compareTimestampsAndDelete(invalidFile, forceDelete, false);
@@ -518,7 +536,7 @@ public class CarbonUpdateUtil {
} else {
invalidDeleteDeltaFiles = updateStatusManager
.getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, false,
- allSegmentFiles);
+ allSegmentFiles, isInvalidFile);
for (CarbonFile invalidFile : invalidDeleteDeltaFiles) {
compareTimestampsAndDelete(invalidFile, forceDelete, false);
@@ -559,6 +577,40 @@ public class CarbonUpdateUtil {
}
/**
+ * This function deletes all the stale carbondata files during clean up before update operation
+ * one scenario is if update operation is ubruptly stopped before updation of table status then
+ * the carbondata file created during update operation is stale file and it will be deleted in
+ * this function in next update operation
+ * @param segment
+ * @param allSegmentFiles
+ * @param updateStatusManager
+ */
+ private static void deleteStaleCarbonDataFiles(LoadMetadataDetails segment,
+ CarbonFile[] allSegmentFiles, SegmentUpdateStatusManager updateStatusManager) {
+ boolean doForceDelete = true;
+ boolean isAbortedFile = true;
+ CarbonFile[] invalidUpdateDeltaFiles = updateStatusManager
+ .getUpdateDeltaFilesList(segment.getLoadName(), false,
+ CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, true, allSegmentFiles,
+ isAbortedFile);
+ // now for each invalid delta file need to check the query execution time out
+ // and then delete.
+ for (CarbonFile invalidFile : invalidUpdateDeltaFiles) {
+ compareTimestampsAndDelete(invalidFile, doForceDelete, false);
+ }
+ // do the same for the index files.
+ CarbonFile[] invalidIndexFiles = updateStatusManager
+ .getUpdateDeltaFilesList(segment.getLoadName(), false,
+ CarbonCommonConstants.UPDATE_INDEX_FILE_EXT, true, allSegmentFiles,
+ isAbortedFile);
+ // now for each invalid index file need to check the query execution time out
+ // and then delete.
+ for (CarbonFile invalidFile : invalidIndexFiles) {
+ compareTimestampsAndDelete(invalidFile, doForceDelete, false);
+ }
+ }
+
+ /**
* This will tell whether the max query timeout has been expired or not.
* @param fileTimestamp
* @return
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b2139cab/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index df7eedd..e0e7b70 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -469,7 +469,7 @@ public class SegmentUpdateStatusManager {
*/
public CarbonFile[] getUpdateDeltaFilesList(String segmentId, final boolean validUpdateFiles,
final String fileExtension, final boolean excludeOriginalFact,
- CarbonFile[] allFilesOfSegment) {
+ CarbonFile[] allFilesOfSegment, boolean isAbortedFile) {
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
@@ -528,7 +528,12 @@ public class SegmentUpdateStatusManager {
}
} else {
// invalid cases.
- if (Long.compare(timestamp, startTimeStampFinal) < 0) {
+ if (isAbortedFile) {
+ if (Long.compare(timestamp, endTimeStampFinal) > 0) {
+ listOfCarbonFiles.add(eachFile);
+ }
+ } else if (Long.compare(timestamp, startTimeStampFinal) < 0
+ || Long.compare(timestamp, endTimeStampFinal) > 0) {
listOfCarbonFiles.add(eachFile);
}
}
@@ -934,11 +939,14 @@ public class SegmentUpdateStatusManager {
*/
public CarbonFile[] getDeleteDeltaInvalidFilesList(final String segmentId,
final SegmentUpdateDetails block, final boolean needCompleteList,
- CarbonFile[] allSegmentFiles) {
+ CarbonFile[] allSegmentFiles, boolean isAbortedFile) {
final long deltaStartTimestamp =
getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
+ final long deltaEndTimestamp =
+ getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
+
List<CarbonFile> files =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
@@ -956,9 +964,16 @@ public class SegmentUpdateStatusManager {
long timestamp = CarbonUpdateUtil.getTimeStampAsLong(
CarbonTablePath.DataFileUtil.getTimeStampFromDeleteDeltaFile(fileName));
- if (block.getBlockName().equalsIgnoreCase(blkName) && (
- Long.compare(timestamp, deltaStartTimestamp) < 0)) {
- files.add(eachFile);
+ if (block.getBlockName().equalsIgnoreCase(blkName)) {
+
+ if (isAbortedFile) {
+ if (Long.compare(timestamp, deltaEndTimestamp) > 0) {
+ files.add(eachFile);
+ }
+ } else if (Long.compare(timestamp, deltaStartTimestamp) < 0
+ || Long.compare(timestamp, deltaEndTimestamp) > 0) {
+ files.add(eachFile);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b2139cab/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 5d7a09f..600b1c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1701,6 +1701,16 @@ public final class CarbonUtil {
&& blockTimeStamp < invalidBlockVOForSegmentId.getUpdateDeltaStartTimestamp()))) {
return true;
}
+ // aborted files case.
+ if (invalidBlockVOForSegmentId.getLatestUpdateTimestamp() != null
+ && blockTimeStamp > invalidBlockVOForSegmentId.getLatestUpdateTimestamp()) {
+ return true;
+ }
+ // for 1st time starttime stamp will be empty so need to consider fact time stamp.
+ if (null == invalidBlockVOForSegmentId.getUpdateDeltaStartTimestamp()
+ && blockTimeStamp > invalidBlockVOForSegmentId.getFactTimestamp()) {
+ return true;
+ }
}
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b2139cab/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index fdc2cc3..12fc5c1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -375,8 +375,10 @@ public final class CarbonLoaderUtil {
}
// reading the start time of data load.
- long loadStartTime = CarbonUpdateUtil.readCurrentTime();
- model.setFactTimeStamp(loadStartTime);
+ if (model.getFactTimeStamp() == 0) {
+ long loadStartTime = CarbonUpdateUtil.readCurrentTime();
+ model.setFactTimeStamp(loadStartTime);
+ }
CarbonLoaderUtil
.populateNewLoadMetaEntry(newLoadMetaEntry, status, model.getFactTimeStamp(), false);
boolean entryAdded =
[07/50] [abbrv] carbondata git commit:
[CARBONDATA-2075][CARBONDATA-1516] It should throw exception when drop
datamap and the table or datamap not exist
Posted by ra...@apache.org.
[CARBONDATA-2075][CARBONDATA-1516] It should throw exception when drop datamap and the table or datamap not exist
error 1:
It should throw no such datamap exception when drop datamap and the datamap does not exist
not IllegalArgumentException or sys.err
It should throw NoSuchDataMapException that extends MalformedCarbonCommandException
error 2:
fix error: drop datamap should throw exception when carbonTable.get.getTableInfo.getDataMapSchemaList.size() == 0
This closes #1858
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/94011c35
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/94011c35
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/94011c35
Branch: refs/heads/fgdatamap
Commit: 94011c35fdeadf48180d9bfafbd9125242d14d3d
Parents: c630b7e
Author: xubo245 <60...@qq.com>
Authored: Thu Jan 25 14:47:34 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Jan 31 08:09:57 2018 +0530
----------------------------------------------------------------------
.../preaggregate/TestPreAggregateDrop.scala | 8 +-
.../timeseries/TestTimeSeriesDropSuite.scala | 108 +++++++++++++++++++
.../testsuite/datamap/TestDataMapCommand.scala | 3 -
.../spark/exception/NoSuchDataMapException.java | 33 ++++++
.../datamap/CarbonDropDataMapCommand.scala | 18 ++--
5 files changed, 156 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
index 147cb6d..1138adf 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
@@ -20,12 +20,12 @@ package org.apache.carbondata.integration.spark.testsuite.preaggregate
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.spark.exception.NoSuchDataMapException
+
class TestPreAggregateDrop extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
sql("drop table if exists maintable")
- sql("drop datamap if exists preagg1 on table maintable")
- sql("drop datamap if exists preagg2 on table maintable")
sql("create table maintable (a string, b string, c string) stored by 'carbondata'")
}
@@ -51,7 +51,7 @@ class TestPreAggregateDrop extends QueryTest with BeforeAndAfterAll {
}
test("drop datamap which is not existed") {
- intercept[RuntimeException] {
+ intercept[NoSuchDataMapException] {
sql("drop datamap newpreagg on table maintable")
}
}
@@ -101,8 +101,6 @@ class TestPreAggregateDrop extends QueryTest with BeforeAndAfterAll {
override def afterAll() {
sql("drop table if exists maintable")
sql("drop table if exists maintable1")
- sql("drop datamap if exists preagg1 on table maintable")
- sql("drop datamap if exists preagg2 on table maintable")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
new file mode 100644
index 0000000..f6d41fb
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+
+class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
+
+ override def beforeAll: Unit = {
+ sql(s"DROP TABLE IF EXISTS mainTable")
+ sql(
+ """
+ | CREATE TABLE mainTable(
+ | dataTime timestamp,
+ | name string,
+ | city string,
+ | age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ }
+
+ test("test timeseries drop datamap 1: drop datamap should throw exception if no datamap") {
+ // DROP DATAMAP DataMapName if the DataMapName not exists
+ checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
+ val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+ }
+ assert(e.getMessage.contains("Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap 2: drop datamap should SUCCESS if haveIF EXISTS") {
+ // DROP DATAMAP DataMapName if the DataMapName not exists
+ checkExistence(sql("show datamap on table mainTable"), false, "agg1_month")
+ try {
+ sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTable")
+ assert(true)
+ } catch {
+ case e: Exception =>
+ println(e)
+ assert(false)
+ }
+ }
+
+ test("test timeseries drop datamap 3: drop datamap should throw proper exception") {
+ sql(
+ """create datamap agg1 on table mainTable
+ |using 'preaggregate'
+ |DMPROPERTIES (
+ | 'timeseries.eventTime'='dataTime',
+ | 'timeseries.hierarchy'='month=1,year=1')
+ |as select dataTime, sum(age) from mainTable
+ |group by dataTime
+ """.stripMargin)
+
+ // Before DROP DATAMAP
+ checkExistence(sql("show datamap on table mainTable"), true, "agg1_month", "agg1_year")
+
+ // DROP DATAMAP DataMapName
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+ checkExistence(sql("show datamap on table mainTable"), false, "agg1_month")
+ val e: Exception = intercept[MalformedCarbonCommandException] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+ }
+ assert(e.getMessage.contains("Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap: drop datamap should throw exception if table not exist") {
+ // DROP DATAMAP DataMapName if the DataMapName not exists and
+ checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
+ val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTableNotExist")
+ }
+ assert(e.getMessage.contains(
+ "Dropping datamap agg1_month failed: Table or view 'maintablenotexist' not found "))
+ }
+
+ test("test timeseries drop datamap: should throw exception if table not exist with IF EXISTS") {
+ // DROP DATAMAP DataMapName if the DataMapName not exists
+ // DROP DATAMAP should throw exception if table not exist, even though there is IF EXISTS"
+ checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
+ val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTableNotExist")
+ }
+ assert(e.getMessage.contains(
+ "Dropping datamap agg1_month failed: Table or view 'maintablenotexist' not found "))
+ }
+
+ override def afterAll: Unit = {
+ sql(s"DROP TABLE IF EXISTS mainTable")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index 0c38239..0860da1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -96,7 +96,6 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
"true")
- sql("drop datamap if exists datamap_hiveMetaStoreTable on table hiveMetaStoreTable")
sql("drop table if exists hiveMetaStoreTable")
sql("create table hiveMetaStoreTable (a string, b string, c string) stored by 'carbondata'")
@@ -121,7 +120,6 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
"true")
- sql("drop datamap if exists datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1")
sql("drop table if exists hiveMetaStoreTable_1")
sql("create table hiveMetaStoreTable_1 (a string, b string, c string) stored by 'carbondata'")
@@ -236,7 +234,6 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
test("test preaggregate load for decimal column for hivemetastore") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
- sql("drop datamap if exists uniqdata_agg on table uniqdata")
sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java
new file mode 100644
index 0000000..959e70d
--- /dev/null
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.exception;
+
+/**
+ * if the dataMap does not exist, carbon should throw NoSuchDataMapException
+ */
+public class NoSuchDataMapException extends MalformedCarbonCommandException {
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ public NoSuchDataMapException(String dataMapName, String tableName) {
+ super("Datamap with name " + dataMapName + " does not exist under table " + tableName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 0ad4457..1fa2494 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.execution.command.AtomicRunnableCommand
import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
import org.apache.spark.sql.execution.command.table.CarbonDropTableCommand
-import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.datamap.DataMapStoreManager
@@ -35,6 +34,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.events._
+import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, NoSuchDataMapException}
/**
* Drops the datamap and any related tables associated with the datamap
@@ -72,8 +72,7 @@ case class CarbonDropDataMapCommand(
Some(CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession))
} catch {
case ex: NoSuchTableException =>
- if (!ifExistsSet) throw ex
- else None
+ throw ex
}
if (carbonTable.isDefined && carbonTable.get.getTableInfo.getDataMapSchemaList.size() > 0) {
val dataMapSchema = carbonTable.get.getTableInfo.getDataMapSchemaList.asScala.zipWithIndex.
@@ -110,15 +109,22 @@ case class CarbonDropDataMapCommand(
sparkSession)
OperationListenerBus.getInstance.fireEvent(dropDataMapPostEvent, operationContext)
} else if (!ifExistsSet) {
- throw new IllegalArgumentException(
- s"Datamap with name $dataMapName does not exist under table $tableName")
+ throw new NoSuchDataMapException(dataMapName, tableName)
+ }
+ } else if ((carbonTable.isDefined &&
+ carbonTable.get.getTableInfo.getDataMapSchemaList.size() == 0)) {
+ if (!ifExistsSet) {
+ throw new NoSuchDataMapException(dataMapName, tableName)
}
}
} catch {
+ case e: NoSuchDataMapException =>
+ throw e
case ex: Exception =>
LOGGER.error(ex, s"Dropping datamap $dataMapName failed")
- sys.error(s"Dropping datamap $dataMapName failed: ${ ex.getMessage }")
+ throw new MalformedCarbonCommandException(
+ s"Dropping datamap $dataMapName failed: ${ex.getMessage}")
}
finally {
if (carbonLocks.nonEmpty) {