You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/11/21 18:00:02 UTC
[35/50] [abbrv] carbondata git commit: [CARBONDATA-3087] Improve DESC
FORMATTED output
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala b/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala
index 03ec3a1..563206f 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/sql/GetDataSizeAndIndexSizeTest.scala
@@ -17,7 +17,10 @@
package org.apache.spark.sql
+import java.util.Date
+
import org.apache.spark.sql.test.util.QueryTest
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.scalatest.BeforeAndAfterAll
@@ -59,7 +62,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll {
.filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
assert(res1.length == 2)
- res1.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ res1.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0))
}
test("get data size and index size after major compaction") {
@@ -73,7 +76,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll {
.filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
assert(res2.length == 2)
- res2.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ res2.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0))
}
test("get data size and index size after minor compaction") {
@@ -91,7 +94,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll {
.filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
assert(res3.length == 2)
- res3.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ res3.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0))
}
test("get data size and index size after insert into") {
@@ -105,7 +108,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll {
.filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
assert(res4.length == 2)
- res4.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ res4.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0))
}
test("get data size and index size after insert overwrite") {
@@ -119,7 +122,7 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll {
.filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
assert(res5.length == 2)
- res5.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ res5.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0))
}
test("get data size and index size for empty table") {
@@ -128,15 +131,14 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll {
.filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
assert(res6.length == 2)
- res6.foreach(row => assert(row.getString(1).trim.toLong == 0))
+ res6.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble == 0))
}
test("get last update time for empty table") {
sql("CREATE TABLE tableSize9 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) STORED BY 'org.apache.carbondata.format'")
val res7 = sql("DESCRIBE FORMATTED tableSize9").collect()
- .filter(row => row.getString(0).contains(CarbonCommonConstants.LAST_UPDATE_TIME))
+ .filter(row => row.getString(0).contains("Last Update"))
assert(res7.length == 1)
- res7.foreach(row => assert(row.getString(1).trim.toLong == 0))
}
test("get last update time for unempty table") {
@@ -144,9 +146,8 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE tableSize10 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""")
val res8 = sql("DESCRIBE FORMATTED tableSize10").collect()
- .filter(row => row.getString(0).contains(CarbonCommonConstants.LAST_UPDATE_TIME))
+ .filter(row => row.getString(0).contains("Last Update"))
assert(res8.length == 1)
- res8.foreach(row => assert(row.getString(1).trim.toLong > 0))
}
test("index and datasize for update scenario") {
@@ -160,13 +161,13 @@ class GetDataSizeAndIndexSizeTest extends QueryTest with BeforeAndAfterAll {
.filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
assert(res9.length == 2)
- res9.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ res9.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0))
sql("update tableSize11 set (empno) = (234)").show()
val res10 = sql("DESCRIBE FORMATTED tableSize11").collect()
.filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) ||
row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE))
assert(res10.length == 2)
- res10.foreach(row => assert(row.getString(1).trim.toLong > 0))
+ res10.foreach(row => assert(row.getString(1).trim.substring(0, 2).toDouble > 0))
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index bcca915..4926cd8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
+import org.apache.carbondata.core.constants.SortScopeOptions;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
@@ -37,7 +38,6 @@ import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
import org.apache.carbondata.processing.loading.steps.CarbonRowDataWriterProcessorStepImpl;
import org.apache.carbondata.processing.loading.steps.DataConverterProcessorStepImpl;
import org.apache.carbondata.processing.loading.steps.DataWriterBatchProcessorStepImpl;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
index 03b2645..f69f0af 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.common.constants.LoggerAction;
import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.SortScopeOptions;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
@@ -38,7 +39,6 @@ import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants;
import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
-import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
import org.apache.carbondata.processing.util.CarbonBadRecordUtil;
import org.apache.carbondata.processing.util.TableOptionConstant;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java
deleted file mode 100644
index 7b3c3df..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.sort;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-
-/**
- * Sort scope options
- */
-public class SortScopeOptions {
-
- public static SortScope getSortScope(String sortScope) {
- if (sortScope == null) {
- sortScope = CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT;
- }
- switch (sortScope.toUpperCase()) {
- case "BATCH_SORT":
- return SortScope.BATCH_SORT;
- case "LOCAL_SORT":
- return SortScope.LOCAL_SORT;
- case "GLOBAL_SORT":
- return SortScope.GLOBAL_SORT;
- case "NO_SORT":
- return SortScope.NO_SORT;
- default:
- return SortScope.LOCAL_SORT;
- }
- }
-
- public enum SortScope {
- NO_SORT, BATCH_SORT, LOCAL_SORT, GLOBAL_SORT
- }
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java
index aad9083..af57759 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java
@@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.SortScopeOptions;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.sort.impl.ParallelReadMergeSorterImpl;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 4012774..92c48bc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.SortScopeOptions;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
@@ -48,7 +49,6 @@ import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
import org.apache.log4j.Logger;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index 48993c5..f168796 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -45,7 +45,7 @@ import org.apache.carbondata.processing.store.writer.AbstractFactDataWriter;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.TABLE_BLOCKLET_SIZE;
import static org.apache.carbondata.core.constants.CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB;
import static org.apache.carbondata.core.constants.CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE;
-import static org.apache.carbondata.processing.loading.sort.SortScopeOptions.SortScope.NO_SORT;
+import static org.apache.carbondata.core.constants.SortScopeOptions.SortScope.NO_SORT;
import org.apache.log4j.Logger;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab1070bc/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index b34ed13..437f628 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.constants.LoggerAction;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.SortScopeOptions;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -52,7 +53,6 @@ import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants;
import org.apache.carbondata.processing.loading.model.CarbonDataLoadSchema;
-import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;