You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2018/06/08 11:40:55 UTC
[30/50] [abbrv] carbondata git commit: [CARBONDATA-2538] added filter
while listing files from writer path
[CARBONDATA-2538] added filter while listing files from writer path
1. Added filter to list only index and carbondata files. So even if the lock files are present proper exception can be thrown
2. Updated complex type docs
This closes #2344
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d7773187
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d7773187
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d7773187
Branch: refs/heads/spark-2.3
Commit: d7773187f72c73b7f9514f13bce17de3c552247c
Parents: 8b80b12
Author: kunal642 <ku...@gmail.com>
Authored: Fri May 25 16:21:45 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Tue May 29 10:49:59 2018 +0530
----------------------------------------------------------------------
.../core/metadata/schema/table/CarbonTable.java | 2 +-
.../readcommitter/LatestFilesReadCommittedScope.java | 9 ++++++++-
.../java/org/apache/carbondata/core/util/CarbonUtil.java | 11 ++++++++---
docs/supported-data-types-in-carbondata.md | 2 ++
.../command/table/CarbonDescribeFormattedCommand.scala | 2 +-
5 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 8528d6f..b1ed981 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -892,7 +892,7 @@ public class CarbonTable implements Serializable {
public long size() throws IOException {
- Map<String, Long> dataIndexSize = CarbonUtil.calculateDataIndexSize(this);
+ Map<String, Long> dataIndexSize = CarbonUtil.calculateDataIndexSize(this, true);
Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
if (dataSize == null) {
dataSize = 0L;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java b/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java
index 6106174..14bba65 100644
--- a/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java
+++ b/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.mutate.UpdateVO;
@@ -138,7 +139,13 @@ public class LatestFilesReadCommittedScope implements ReadCommittedScope {
@Override public void takeCarbonIndexFileSnapShot() throws IOException {
// Read the current file Path get the list of indexes from the path.
CarbonFile file = FileFactory.getCarbonFile(carbonFilePath);
- if (file.listFiles().length == 0) {
+ CarbonFile[] files = file.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT) || file.getName()
+ .endsWith(CarbonTablePath.CARBON_DATA_EXT);
+ }
+ });
+ if (files.length == 0) {
// For nonTransactional table, files can be removed at any point of time.
// So cannot assume files will be present
throw new IOException("No files are present in the table location :" + carbonFilePath);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 23d02ef..9ccd772 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2550,7 +2550,8 @@ public final class CarbonUtil {
/**
* This method will calculate the data size and index size for carbon table
*/
- public static Map<String, Long> calculateDataIndexSize(CarbonTable carbonTable)
+ public static Map<String, Long> calculateDataIndexSize(CarbonTable carbonTable,
+ Boolean updateSize)
throws IOException {
Map<String, Long> dataIndexSizeMap = new HashMap<String, Long>();
long dataSize = 0L;
@@ -2565,7 +2566,11 @@ public final class CarbonUtil {
SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
try {
- if (carbonLock.lockWithRetries()) {
+ boolean lockAcquired = true;
+ if (updateSize) {
+ lockAcquired = carbonLock.lockWithRetries();
+ }
+ if (lockAcquired) {
LOGGER.info("Acquired lock for table for table status updation");
String metadataPath = carbonTable.getMetadataPath();
LoadMetadataDetails[] loadMetadataDetails =
@@ -2593,7 +2598,7 @@ public final class CarbonUtil {
}
}
// If it contains old segment, write new load details
- if (needUpdate) {
+ if (needUpdate && updateSize) {
SegmentStatusManager.writeLoadDetailsIntoFile(
CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()),
loadMetadataDetails);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/docs/supported-data-types-in-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/supported-data-types-in-carbondata.md b/docs/supported-data-types-in-carbondata.md
index 6c21508..7260afe 100644
--- a/docs/supported-data-types-in-carbondata.md
+++ b/docs/supported-data-types-in-carbondata.md
@@ -38,6 +38,8 @@
* Complex Types
* arrays: ARRAY``<data_type>``
* structs: STRUCT``<col_name : data_type COMMENT col_comment, ...>``
+
+ **NOTE**: Only 2 level complex type schema is supported for now.
* Other Types
* BOOLEAN
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index 69bb91f..7d15cc1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -96,7 +96,7 @@ private[sql] case class CarbonDescribeFormattedCommand(
val tableComment = tblProps.asScala.getOrElse(CarbonCommonConstants.TABLE_COMMENT, "")
results ++= Seq(("Comment", tableComment, ""))
results ++= Seq(("Table Block Size ", carbonTable.getBlockSizeInMB + " MB", ""))
- val dataIndexSize = CarbonUtil.calculateDataIndexSize(carbonTable)
+ val dataIndexSize = CarbonUtil.calculateDataIndexSize(carbonTable, false)
if (!dataIndexSize.isEmpty) {
results ++= Seq((CarbonCommonConstants.TABLE_DATA_SIZE,
dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE).toString, ""))