You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/06/15 11:50:29 UTC
[24/42] carbondata git commit: MultiClient Load is failing
MultiClient Load is failing
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ef583afe
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ef583afe
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ef583afe
Branch: refs/heads/branch-1.1
Commit: ef583afe6968d1222553810bdc2251cef16f016c
Parents: 735e477
Author: nareshpr <pr...@gmail.com>
Authored: Tue May 30 14:48:10 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Jun 15 13:12:41 2017 +0530
----------------------------------------------------------------------
.../org/apache/carbondata/spark/load/CarbonLoaderUtil.java | 9 ++-------
.../apache/carbondata/spark/rdd/DataManagementFunc.scala | 4 ++--
.../apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../spark/sql/execution/command/carbonTableSchema.scala | 6 +++---
.../scala/org/apache/spark/sql/hive/CarbonMetastore.scala | 2 --
6 files changed, 9 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index a4f15d2..54e12f3 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -58,7 +58,6 @@ import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
import org.apache.carbondata.core.fileoperations.FileWriteOperation;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
@@ -783,14 +782,10 @@ public final class CarbonLoaderUtil {
* This method will get the store location for the given path, segment id and partition id
*
* @param carbonStorePath
- * @param dbName
- * @param tableName
* @param segmentId
*/
- public static void checkAndCreateCarbonDataLocation(String carbonStorePath, String dbName,
- String tableName, String segmentId) {
- CarbonTable carbonTable = CarbonMetadata.getInstance()
- .getCarbonTable(dbName + CarbonCommonConstants.UNDERSCORE + tableName);
+ public static void checkAndCreateCarbonDataLocation(String carbonStorePath,
+ String segmentId, CarbonTable carbonTable) {
CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
index 8039d24..1790ea2 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.execution.command.{CompactionCallableModel, Compacti
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
-import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager}
@@ -59,7 +59,7 @@ object DataManagementFunc {
val sc = sqlContext
// Delete the records based on data
- val table = CarbonMetadata.getInstance.getCarbonTable(databaseName + "_" + tableName)
+ val table = schema.getCarbonTable
val loadMetadataDetailsArray =
SegmentStatusManager.readLoadMetadata(table.getMetaDataFilepath).toList
val resultMap = new CarbonDeleteLoadByDateRDD(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index f159c61..2922365 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -734,7 +734,7 @@ object CarbonDataRDDFactory {
if (!updateModel.isDefined) {
CarbonLoaderUtil.checkAndCreateCarbonDataLocation(storePath,
- carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName, currentLoadCount.toString)
+ currentLoadCount.toString, carbonTable)
}
var loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
var errorMessage: String = "DataLoad failure"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index bbdbe4f..b4720a9 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -764,7 +764,7 @@ object CarbonDataRDDFactory {
if (!updateModel.isDefined) {
CarbonLoaderUtil.checkAndCreateCarbonDataLocation(storePath,
- carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName, currentLoadCount.toString)
+ currentLoadCount.toString, carbonTable)
}
var loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
var errorMessage: String = "DataLoad failure"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 5dd6832..8818c6b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -41,7 +41,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
-import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
@@ -89,7 +89,7 @@ case class AlterTableCompaction(alterTableModel: AlterTableModel) extends Runnab
if (relation == null) {
sys.error(s"Table $databaseName.$tableName does not exist")
}
- if (null == CarbonMetadata.getInstance.getCarbonTable(databaseName + "_" + tableName)) {
+ if (null == relation.tableMeta.carbonTable) {
LOGGER.error(s"alter table failed. table not found: $databaseName.$tableName")
sys.error(s"alter table failed. table not found: $databaseName.$tableName")
}
@@ -352,7 +352,7 @@ case class LoadTable(
if (relation == null) {
sys.error(s"Table $dbName.$tableName does not exist")
}
- if (null == CarbonMetadata.getInstance.getCarbonTable(dbName + "_" + tableName)) {
+ if (null == relation.tableMeta.carbonTable) {
LOGGER.error(s"Data loading failed. table not found: $dbName.$tableName")
LOGGER.audit(s"Data loading failed. table not found: $dbName.$tableName")
sys.error(s"Data loading failed. table not found: $dbName.$tableName")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 1f5736e..954801a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -40,7 +40,6 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
import org.apache.carbondata.core.fileoperations.FileWriteOperation
-import org.apache.carbondata.core.locks.ZookeeperInit
import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.datatype.DataType.DECIMAL
@@ -529,7 +528,6 @@ class CarbonMetastore(conf: RuntimeConfig, val storePath: String) {
case Some(tableMeta) =>
metadata.tablesMeta -= tableMeta
CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
- CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
case None =>
LOGGER.info(s"Metadata does not contain entry for table $tableName in database $dbName")