You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/06 10:00:21 UTC
[04/20] incubator-carbondata git commit: [CARBONDATA-130]Adapt kettle
home when "carbon.kettle.home" configuration is wrong (#889)
[CARBONDATA-130]Adapt kettle home when "carbon.kettle.home" configuration is wrong (#889)
Generally kettle home is inside carbon lib so same can be looked in default
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f9c2e57d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f9c2e57d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f9c2e57d
Branch: refs/heads/master
Commit: f9c2e57d670a51d5d8379f8b9b89a37b6909ce10
Parents: c44bfd3
Author: Gin-zhj <zh...@huawei.com>
Authored: Mon Aug 1 22:20:26 2016 +0800
Committer: Kumar Vishal <ku...@gmail.com>
Committed: Mon Aug 1 19:50:26 2016 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 ++
.../execution/command/carbonTableSchema.scala | 25 ++--------
.../spark/rdd/CarbonDataRDDFactory.scala | 10 +---
.../carbondata/spark/util/CarbonScalaUtil.scala | 51 +++++++++++++++++++-
4 files changed, 60 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f9c2e57d/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index b45d9b4..724698f 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -73,6 +73,10 @@ public final class CarbonCommonConstants {
*/
public static final String STORE_LOCATION_DEFAULT_VAL = "../carbon.store";
/**
+ * the folder name of kettle home path
+ */
+ public static final String KETTLE_HOME_NAME = "carbonplugins";
+ /**
* CARDINALITY_INCREMENT_DEFAULT_VALUE
*/
public static final int CARDINALITY_INCREMENT_VALUE_DEFAULT_VAL = 10;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f9c2e57d/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 1dd066f..cc08604 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -1230,14 +1230,8 @@ private[sql] case class AlterTableCompaction(alterTableModel: AlterTableModel) e
carbonLoadModel.setStorePath(relation.cubeMeta.storePath)
val partitioner = relation.cubeMeta.partitioner
+ val kettleHomePath = CarbonScalaUtil.getKettleHome(sqlContext)
- var kettleHomePath = sqlContext.getConf("carbon.kettle.home", null)
- if (null == kettleHomePath) {
- kettleHomePath = CarbonProperties.getInstance.getProperty("carbon.kettle.home")
- }
- if (kettleHomePath == null) {
- sys.error(s"carbon.kettle.home is not set")
- }
var storeLocation = CarbonProperties.getInstance
.getProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
System.getProperty("java.io.tmpdir")
@@ -1524,13 +1518,7 @@ private[sql] case class LoadCube(
storeLocation = storeLocation + "/carbonstore/" + System.nanoTime()
val columinar = sqlContext.getConf("carbon.is.columnar.storage", "true").toBoolean
- var kettleHomePath = sqlContext.getConf("carbon.kettle.home", null)
- if (null == kettleHomePath) {
- kettleHomePath = CarbonProperties.getInstance.getProperty("carbon.kettle.home")
- }
- if (kettleHomePath == null) {
- sys.error(s"carbon.kettle.home is not set")
- }
+ val kettleHomePath = CarbonScalaUtil.getKettleHome(sqlContext)
val delimiter = partionValues.getOrElse("delimiter", ",")
val quoteChar = partionValues.getOrElse("quotechar", "\"")
@@ -1721,13 +1709,8 @@ private[sql] case class LoadAggregationTable(
System.getProperty("java.io.tmpdir"))
storeLocation = storeLocation + "/carbonstore/" + System.currentTimeMillis()
val columinar = sqlContext.getConf("carbon.is.columnar.storage", "true").toBoolean
- var kettleHomePath = sqlContext.getConf("carbon.kettle.home", null)
- if (null == kettleHomePath) {
- kettleHomePath = CarbonProperties.getInstance.getProperty("carbon.kettle.home")
- }
- if (kettleHomePath == null) {
- sys.error(s"carbon.kettle.home is not set")
- }
+ val kettleHomePath = CarbonScalaUtil.getKettleHome(sqlContext)
+
CarbonDataRDDFactory.loadCarbonData(
sqlContext,
carbonLoadModel,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f9c2e57d/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 49e6702..f554a35 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -52,7 +52,7 @@ import org.carbondata.spark._
import org.carbondata.spark.load._
import org.carbondata.spark.merger.CarbonDataMergerUtil
import org.carbondata.spark.splits.TableSplit
-import org.carbondata.spark.util.{CarbonQueryUtil, LoadMetadataUtil}
+import org.carbondata.spark.util.{CarbonQueryUtil, CarbonScalaUtil, LoadMetadataUtil}
/**
@@ -255,13 +255,7 @@ object CarbonDataRDDFactory extends Logging {
)
storeLocation = storeLocation + "/carbonstore/" + System.currentTimeMillis()
val columinar = sqlContext.getConf("carbon.is.columnar.storage", "true").toBoolean
- var kettleHomePath = sqlContext.getConf("carbon.kettle.home", null)
- if (null == kettleHomePath) {
- kettleHomePath = CarbonProperties.getInstance.getProperty("carbon.kettle.home")
- }
- if (kettleHomePath == null) {
- sys.error(s"carbon.kettle.home is not set")
- }
+ val kettleHomePath = CarbonScalaUtil.getKettleHome(sqlContext)
CarbonDataRDDFactory.loadCarbonData(
sqlContext,
carbonLoadModel,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f9c2e57d/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
index b3effd3..87dd0ce 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
@@ -17,8 +17,11 @@
package org.carbondata.spark.util
+import java.io.File
+
import scala.collection.JavaConverters._
+import org.apache.spark.Logging
import org.apache.spark.sql._
import org.apache.spark.sql.execution.command.Level
import org.apache.spark.sql.hive.{CarbonMetaData, DictionaryMap}
@@ -28,9 +31,11 @@ import org.carbondata.core.carbon.metadata.datatype.DataType
import org.carbondata.core.carbon.metadata.encoder.Encoding
import org.carbondata.core.carbon.metadata.schema.table.CarbonTable
import org.carbondata.core.constants.CarbonCommonConstants
+import org.carbondata.core.datastorage.store.impl.FileFactory
+import org.carbondata.core.util.CarbonProperties
import org.carbondata.query.expression.{DataType => CarbonDataType}
-object CarbonScalaUtil {
+object CarbonScalaUtil extends Logging {
def convertSparkToCarbonDataType(
dataType: org.apache.spark.sql.types.DataType): CarbonDataType = {
dataType match {
@@ -142,4 +147,48 @@ object CarbonScalaUtil {
}
}
+ def getKettleHome(sqlContext: SQLContext): String = {
+ var kettleHomePath = sqlContext.getConf("carbon.kettle.home", null)
+ if (null == kettleHomePath) {
+ kettleHomePath = CarbonProperties.getInstance.getProperty("carbon.kettle.home")
+ }
+ if (kettleHomePath != null) {
+ val sparkMaster = sqlContext.sparkContext.getConf.get("spark.master").toLowerCase()
+ // get spark master, if local, need to correct the kettle home
+ // e.g: --master local, the executor running in local machine
+ if (sparkMaster.startsWith("local")) {
+ val kettleHomeFileType = FileFactory.getFileType(kettleHomePath)
+ val kettleHomeFile = FileFactory.getCarbonFile(kettleHomePath, kettleHomeFileType)
+ // check if carbon.kettle.home path is exists
+ if (!kettleHomeFile.exists()) {
+ // get the path of this class
+ // e.g: file:/srv/bigdata/install/spark/sparkJdbc/carbonlib/carbon-
+ // xxx.jar!/org/carbondata/spark/rdd/
+ var jarFilePath = this.getClass.getResource("").getPath
+ val endIndex = jarFilePath.indexOf(".jar!") + 4
+ // get the jar file path
+ // e.g: file:/srv/bigdata/install/spark/sparkJdbc/carbonlib/carbon-*.jar
+ jarFilePath = jarFilePath.substring(0, endIndex)
+ val jarFileType = FileFactory.getFileType(jarFilePath)
+ val jarFile = FileFactory.getCarbonFile(jarFilePath, jarFileType)
+ // get the parent folder of the jar file
+ // e.g:file:/srv/bigdata/install/spark/sparkJdbc/carbonlib
+ val carbonLibPath = jarFile.getParentFile.getPath
+ // find the kettle home under the previous folder
+ // e.g:file:/srv/bigdata/install/spark/sparkJdbc/carbonlib/cabonplugins
+ kettleHomePath = carbonLibPath + File.separator + CarbonCommonConstants.KETTLE_HOME_NAME
+ logInfo(s"carbon.kettle.home path is not exists, reset it as $kettleHomePath")
+ val newKettleHomeFileType = FileFactory.getFileType(kettleHomePath)
+ val newKettleHomeFile = FileFactory.getCarbonFile(kettleHomePath, newKettleHomeFileType)
+ // check if the found kettle home exists
+ if (!newKettleHomeFile.exists()) {
+ sys.error("Kettle home not found. Failed to reset carbon.kettle.home")
+ }
+ }
+ }
+ } else {
+ sys.error("carbon.kettle.home is not set")
+ }
+ kettleHomePath
+ }
}