You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ak...@apache.org on 2020/04/07 08:22:14 UTC
[carbondata] branch master updated: [CARBONDATA-3766] Fixed desc
formatted and show segment data size issues
This is an automated email from the ASF dual-hosted git repository.
akashrn5 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new e643747 [CARBONDATA-3766] Fixed desc formatted and show segment data size issues
e643747 is described below
commit e6437479fa11eef9a8fc03c2572905bee458c5ad
Author: kunal642 <ku...@gmail.com>
AuthorDate: Thu Mar 26 14:55:34 2020 +0530
[CARBONDATA-3766] Fixed desc formatted and show segment data size issues
Why is this PR needed?
Show segments shows the datasize as index size for non carbon segments in show segments command.
What changes were proposed in this PR?
Take data size from indexsize variable and show in show segments as non carbon segmnets do not have
index files. Index Size wil be shown as 0.
This closes #3680
---
.../carbondata/core/metadata/SegmentFileStore.java | 3 ++-
.../java/org/apache/carbondata/core/util/CarbonUtil.java | 16 ++++++++++++----
.../scala/org/apache/carbondata/api/CarbonStore.scala | 2 +-
.../spark/testsuite/addsegment/AddSegmentTestCase.scala | 2 ++
4 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index afb76bd..7aa2843 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -527,10 +527,11 @@ public class SegmentFileStore {
for (LoadMetadataDetails detail : listOfLoadFolderDetailsArray) {
// if the segments is in the list of marked for delete then update the status.
if (segmentId.equals(detail.getLoadName())) {
+ detail.setLoadEndTime(System.currentTimeMillis());
detail.setSegmentFile(segmentFile);
if (segmentStatus != null) {
HashMap<String, Long> dataSizeAndIndexSize =
- CarbonUtil.getDataSizeAndIndexSize(segmentFileStore);
+ CarbonUtil.getDataSizeAndIndexSize(segmentFileStore, detail.isCarbonFormat());
detail.setDataSize(
dataSizeAndIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE)
.toString());
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index d00da15..7917ddd 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2514,7 +2514,8 @@ public final class CarbonUtil {
}
// Get the total size of carbon data and the total size of carbon index
- public static HashMap<String, Long> getDataSizeAndIndexSize(SegmentFileStore fileStore)
+ public static HashMap<String, Long> getDataSizeAndIndexSize(SegmentFileStore fileStore,
+ boolean isCarbonSegment)
throws IOException {
long carbonDataSize = 0L;
long carbonIndexSize = 0L;
@@ -2532,8 +2533,15 @@ public final class CarbonUtil {
}
}
}
- dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE, carbonDataSize);
- dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, carbonIndexSize);
+ if (isCarbonSegment) {
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE, carbonDataSize);
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, carbonIndexSize);
+ } else {
+ // In case of non-carbon segments all the dataSize is collected in carbonIndexSize because of
+ // listing behaviour of the SegmentFileStore.
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE, carbonIndexSize);
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, 0L);
+ }
return dataAndIndexSize;
}
@@ -2582,7 +2590,7 @@ public final class CarbonUtil {
Segment segment) throws IOException {
if (segment.getSegmentFileName() != null) {
SegmentFileStore fileStore = new SegmentFileStore(tablePath, segment.getSegmentFileName());
- return getDataSizeAndIndexSize(fileStore);
+ return getDataSizeAndIndexSize(fileStore, segment.isCarbonSegment());
} else {
return getDataSizeAndIndexSize(tablePath, segment.getSegmentNo());
}
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index 5571656..275d260 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -123,7 +123,7 @@ object CarbonStore {
// If the added segment is other than carbon segment then we can only display the data
// size and not index size, we can get the data size from table status file directly
if (!load.getFileFormat.isCarbonFormat) {
- (if (load.getIndexSize == null) -1L else load.getIndexSize.toLong, -1L)
+ (if (load.getDataSize == null) -1L else load.getDataSize.toLong, -1L)
} else {
(if (load.getDataSize == null) -1L else load.getDataSize.toLong,
if (load.getIndexSize == null) -1L else load.getIndexSize.toLong)
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
index 13472cc..6c41abd 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
@@ -230,7 +230,9 @@ class AddSegmentTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select count(*) from addsegment1"), Seq(Row(20)))
sql("show segments for table addsegment1").show(100, false)
val showSeg = sql("show segments for table addsegment1").collectAsList()
+ val descFormattedSize = sql("desc formatted addsegment1").collect().filter(_.get(0).toString.startsWith("Table Data Size")).head.get(1).toString
val size = getDataSize(newPath)
+ assert(descFormattedSize.split("KB")(0).toDouble > 0.0d)
assert(showSeg.get(0).get(6).toString.equalsIgnoreCase(size))
assert(showSeg.get(0).get(7).toString.equalsIgnoreCase("NA"))
FileFactory.deleteAllFilesOfDir(new File(newPath))