You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/10/28 00:40:36 UTC

[1/2] incubator-carbondata git commit: fix issue carbondata-339

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 5a97fc795 -> ab9c6c083


fix issue carbondata-339


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/64d4d6da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/64d4d6da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/64d4d6da

Branch: refs/heads/master
Commit: 64d4d6daaf6e8adede6cfffe94221d20f365631c
Parents: 5a97fc7
Author: hseagle <hs...@gmail.com>
Authored: Thu Oct 27 10:55:53 2016 +0800
Committer: hseagle <hs...@gmail.com>
Committed: Thu Oct 27 10:55:53 2016 +0800

----------------------------------------------------------------------
 .../carbondata/spark/util/GlobalDictionaryUtil.scala    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/64d4d6da/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 09c32c8..a1f4400 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -742,13 +742,13 @@ object GlobalDictionaryUtil extends Logging {
    */
   def generateGlobalDictionary(sqlContext: SQLContext,
                                carbonLoadModel: CarbonLoadModel,
-                               hdfsLocation: String,
+                               storePath: String,
                                dataFrame: Option[DataFrame] = None): Unit = {
     try {
       val table = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getAbsoluteTableIdentifier
         .getCarbonTableIdentifier
       // create dictionary folder if not exists
-      val carbonTablePath = CarbonStorePath.getCarbonTablePath(hdfsLocation, table)
+      val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, table)
       val dictfolderPath = carbonTablePath.getMetadataDirectoryPath
       // columns which need to generate global dictionary file
       val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
@@ -777,7 +777,7 @@ object GlobalDictionaryUtil extends Logging {
         if (colDictFilePath != null) {
           // generate predefined dictionary
           generatePredefinedColDictionary(colDictFilePath, table,
-            dimensions, carbonLoadModel, sqlContext, hdfsLocation, dictfolderPath)
+            dimensions, carbonLoadModel, sqlContext, storePath, dictfolderPath)
         }
         if (headers.length > df.columns.length) {
           val msg = "The number of columns in the file header do not match the number of " +
@@ -792,7 +792,7 @@ object GlobalDictionaryUtil extends Logging {
           // select column to push down pruning
           df = df.select(requireColumnNames.head, requireColumnNames.tail: _*)
           val model = createDictionaryLoadModel(carbonLoadModel, table, requireDimension,
-            hdfsLocation, dictfolderPath, false)
+            storePath, dictfolderPath, false)
           // combine distinct value in a block and partition by column
           val inputRDD = new CarbonBlockDistinctValuesCombineRDD(df.rdd, model)
             .partitionBy(new ColumnPartitioner(model.primDimensions.length))
@@ -815,7 +815,7 @@ object GlobalDictionaryUtil extends Logging {
               dimDataframe = dimDataframe.select(requireColumnNamesForDim.head,
                 requireColumnNamesForDim.tail: _*)
               val modelforDim = createDictionaryLoadModel(carbonLoadModel, table,
-                requireDimensionForDim, hdfsLocation, dictfolderPath, false)
+                requireDimensionForDim, storePath, dictfolderPath, false)
               val inputRDDforDim = new CarbonBlockDistinctValuesCombineRDD(
                 dimDataframe.rdd, modelforDim)
                 .partitionBy(new ColumnPartitioner(modelforDim.primDimensions.length))
@@ -842,7 +842,7 @@ object GlobalDictionaryUtil extends Logging {
             pruneDimensions(dimensions, headers, headers)
           if (requireDimension.nonEmpty) {
             val model = createDictionaryLoadModel(carbonLoadModel, table, requireDimension,
-              hdfsLocation, dictfolderPath, false)
+              storePath, dictfolderPath, false)
             // check if dictionary files contains bad record
             val accumulator = sqlContext.sparkContext.accumulator(0)
             // read local dictionary file, and group by key


[2/2] incubator-carbondata git commit: [carbondata-339] Align storePath name in generateGlobalDictionary() of GlobalDictionaryUtil.scala This closes #261

Posted by ja...@apache.org.
[carbondata-339] Align storePath name in generateGlobalDictionary() of GlobalDictionaryUtil.scala This closes #261


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ab9c6c08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ab9c6c08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ab9c6c08

Branch: refs/heads/master
Commit: ab9c6c083e53ccd9b29881fe0fa1a727e76be6e9
Parents: 5a97fc7 64d4d6d
Author: jackylk <ja...@huawei.com>
Authored: Fri Oct 28 08:40:11 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Fri Oct 28 08:40:11 2016 +0800

----------------------------------------------------------------------
 .../carbondata/spark/util/GlobalDictionaryUtil.scala    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------