You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/12/19 17:39:16 UTC
carbondata git commit: [CARBONDATA-1872][API] Clean up unused
constant in CarbonCommonConstant
Repository: carbondata
Updated Branches:
refs/heads/master b8a02f391 -> b0d4de94e
[CARBONDATA-1872][API] Clean up unused constant in CarbonCommonConstant
This closes #1631
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b0d4de94
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b0d4de94
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b0d4de94
Branch: refs/heads/master
Commit: b0d4de94ebf6bd9f5851c14f0d4a11bf2d74da23
Parents: b8a02f3
Author: Jacky Li <ja...@qq.com>
Authored: Thu Dec 7 19:40:45 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Dec 19 23:08:58 2017 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 161 +------------------
.../core/mutate/CarbonUpdateUtil.java | 1 -
.../core/statusmanager/LoadMetadataDetails.java | 19 ---
.../sql/test/Spark2TestQueryExecutor.scala | 6 +-
.../store/CarbonFactDataHandlerColumnar.java | 53 ------
5 files changed, 6 insertions(+), 234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index f67b0c5..b8a998c 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -46,14 +46,6 @@ public final class CarbonCommonConstants {
* byte to KB conversion factor
*/
public static final int BYTE_TO_KB_CONVERSION_FACTOR = 1024;
- /**
- * BYTE_ENCODING
- */
- public static final String BYTE_ENCODING = "ISO-8859-1";
- /**
- * measure meta data file name
- */
- public static final String MEASURE_METADATA_FILE_NAME = "/msrMetaData_";
/**
* set the segment ids to query from the table
@@ -179,22 +171,6 @@ public final class CarbonCommonConstants {
public static final int BYTEBUFFER_SIZE = 24 * 1024;
/**
- * Average constant
- */
- public static final String AVERAGE = "avg";
- /**
- * Count constant
- */
- public static final String COUNT = "count";
- /**
- * SUM
- */
- public static final String SUM = "sum";
- /**
- * DUMMY aggregation function
- */
- public static final String DUMMY = "dummy";
- /**
* MEMBER_DEFAULT_VAL
*/
public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!";
@@ -234,18 +210,10 @@ public final class CarbonCommonConstants {
*/
public static final String CARBON_BADRECORDS_LOC_DEFAULT_VAL = "";
/**
- * HIERARCHY_FILE_EXTENSION
- */
- public static final String HIERARCHY_FILE_EXTENSION = ".hierarchy";
- /**
* SORT_TEMP_FILE_LOCATION
*/
public static final String SORT_TEMP_FILE_LOCATION = "sortrowtmp";
/**
- * CARBON_RESULT_SIZE_DEFAULT
- */
- public static final String LEVEL_FILE_EXTENSION = ".level";
- /**
* FACT_FILE_EXT
*/
public static final String FACT_FILE_EXT = ".carbondata";
@@ -260,35 +228,6 @@ public final class CarbonCommonConstants {
*/
public static final String UPDATE_DELTA_FILE_EXT = FACT_FILE_EXT;
/**
- * MEASUREMETADATA_FILE_EXT
- */
- public static final String MEASUREMETADATA_FILE_EXT = ".msrmetadata";
- /**
- * GRAPH_ROWSET_SIZE
- */
- @CarbonProperty
- public static final String GRAPH_ROWSET_SIZE = "carbon.graph.rowset.size";
- /**
- * GRAPH_ROWSET_SIZE_DEFAULT
- */
- public static final String GRAPH_ROWSET_SIZE_DEFAULT = "500";
- /**
- * Comment for <code>TYPE_MYSQL</code>
- */
- public static final String TYPE_MYSQL = "MYSQL";
- /**
- * Comment for <code>TYPE_MSSQL</code>
- */
- public static final String TYPE_MSSQL = "MSSQL";
- /**
- * Comment for <code>TYPE_ORACLE</code>
- */
- public static final String TYPE_ORACLE = "ORACLE";
- /**
- * Comment for <code>TYPE_SYBASE</code>
- */
- public static final String TYPE_SYBASE = "SYBASE";
- /**
* SORT_INTERMEDIATE_FILES_LIMIT
*/
@CarbonProperty
@@ -298,10 +237,6 @@ public final class CarbonCommonConstants {
*/
public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
/**
- * BAD_RECORD_KEY_VALUE
- */
- public static final String BAD_RECORD_KEY = "BADRECORD";
- /**
* MERGERD_EXTENSION
*/
public static final String MERGERD_EXTENSION = ".merge";
@@ -311,15 +246,6 @@ public final class CarbonCommonConstants {
@CarbonProperty
public static final String SORT_FILE_BUFFER_SIZE = "carbon.sort.file.buffer.size";
/**
- * no.of records after which counter to be printed
- */
- @CarbonProperty
- public static final String DATA_LOAD_LOG_COUNTER = "carbon.load.log.counter";
- /**
- * DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER
- */
- public static final String DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER = "500000";
- /**
* SORT_FILE_WRITE_BUFFER_SIZE
*/
@CarbonProperty
@@ -405,18 +331,6 @@ public final class CarbonCommonConstants {
*/
public static final String SEMICOLON_SPC_CHARACTER = ";#!@:SEMIC:@!#;";
/**
- * AMPERSAND_SPC_CHARATER
- */
- public static final String AMPERSAND_SPC_CHARACTER = "&#!@:AMPER:@!#&";
- /**
- * ATTHERATE_SPC_CHARATER
- */
- public static final String COMA_SPC_CHARACTER = ",#!:COMA:!#,";
- /**
- * HYPHEN_SPC_CHARACTER
- */
- public static final String HYPHEN_SPC_CHARACTER = "-#!:HYPHEN:!#-";
- /**
* SORT_TEMP_FILE_EXT
*/
public static final String SORT_TEMP_FILE_EXT = ".sorttemp";
@@ -485,27 +399,10 @@ public final class CarbonCommonConstants {
@CarbonProperty
public static final String STORE_LOCATION_TEMP_PATH = "carbon.tempstore.location";
/**
- * IS_COLUMNAR_STORAGE_DEFAULTVALUE
- */
- public static final String IS_COLUMNAR_STORAGE_DEFAULTVALUE = "true";
- /**
- * DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE
- */
- public static final String DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE = "1";
- /**
* IS_FULLY_FILLED_BITS_DEFAULT_VALUE
*/
public static final String IS_FULLY_FILLED_BITS_DEFAULT_VALUE = "true";
/**
- * IS_INT_BASED_INDEXER
- */
- @CarbonProperty
- public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK = "aggregate.columnar.keyblock";
- /**
- * IS_INT_BASED_INDEXER_DEFAULTVALUE
- */
- public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK_DEFAULTVALUE = "true";
- /**
* ENABLE_QUERY_STATISTICS
*/
@CarbonProperty
@@ -541,10 +438,6 @@ public final class CarbonCommonConstants {
*/
public static final String UNDERSCORE = "_";
/**
- * DASH
- */
- public static final String DASH = "-";
- /**
* POINT
*/
public static final String POINT = ".";
@@ -621,16 +514,6 @@ public final class CarbonCommonConstants {
public static final String TIMESTAMP = "Timestamp";
public static final String ARRAY = "array";
public static final String STRUCT = "struct";
- public static final String FROM = "from";
- /**
- * FACT_UPDATE_EXTENSION.
- */
- public static final String FACT_UPDATE_EXTENSION = ".carbondata_update";
- public static final String FACT_DELETE_EXTENSION = "_delete";
- /**
- * MARKED_FOR_UPDATION
- */
- public static final String FACT_FILE_UPDATED = "update";
/**
* MAX_QUERY_EXECUTION_TIME
*/
@@ -663,17 +546,11 @@ public final class CarbonCommonConstants {
*/
@CarbonProperty
public static final String CARBON_MAX_DRIVER_LRU_CACHE_SIZE = "carbon.max.driver.lru.cache.size";
- public static final String POSITION_REFERENCE = "positionReference";
/**
* implicit column which will be added to each carbon table
*/
public static final String POSITION_ID = "positionId";
/**
- * max driver lru cache size upto which lru cache will be loaded in memory
- */
- @CarbonProperty
- public static final String CARBON_MAX_LEVEL_CACHE_SIZE = "carbon.max.level.cache.size";
- /**
* max executor lru cache size upto which lru cache will be loaded in memory
*/
@CarbonProperty
@@ -1027,16 +904,6 @@ public final class CarbonCommonConstants {
public static final String UPDATE_INDEX_FILE_EXT = ".carbonindex";
/**
- * Key word for true
- */
- public static final String KEYWORD_TRUE = "TRUE";
-
- /**
- * Key word for false
- */
- public static final String KEYWORD_FALSE = "FALSE";
-
- /**
* hyphen
*/
public static final String HYPHEN = "-";
@@ -1047,11 +914,6 @@ public final class CarbonCommonConstants {
public static final String UPDATED_COL_EXTENSION = "-updatedColumn";
/**
- * appending the key to differentiate the update flow with insert flow.
- */
- public static final String RDDUTIL_UPDATE_KEY = "UPDATE_";
-
- /**
* to determine to use the rdd persist or not.
*/
@CarbonProperty
@@ -1099,10 +961,6 @@ public final class CarbonCommonConstants {
public static final String CARBON_DATA_FILE_DEFAULT_VERSION = "V3";
/**
- * data file version header
- */
- public static final String CARBON_DATA_VERSION_HEADER = "CARBONDATAVERSION#";
- /**
* Maximum no of column supported
*/
public static final int DEFAULT_MAX_NUMBER_OF_COLUMNS = 20000;
@@ -1200,12 +1058,13 @@ public final class CarbonCommonConstants {
@CarbonProperty
public static final String LOAD_BATCH_SORT_SIZE_INMB = "carbon.load.batch.sort.size.inmb";
public static final String LOAD_BATCH_SORT_SIZE_INMB_DEFAULT = "0";
- @CarbonProperty
+
/**
* The Number of partitions to use when shuffling data for sort. If user don't configurate or
* configurate it less than 1, it uses the number of map tasks as reduce tasks. In general, we
* recommend 2-3 tasks per CPU core in your cluster.
*/
+ @CarbonProperty
public static final String LOAD_GLOBAL_SORT_PARTITIONS = "carbon.load.global.sort.partitions";
public static final String LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT = "0";
@@ -1232,11 +1091,6 @@ public final class CarbonCommonConstants {
public static final String IS_DRIVER_INSTANCE = "is.driver.instance";
/**
- * maximum length of column
- */
- public static final int DEFAULT_COLUMN_LENGTH = 100000;
-
- /**
* property for enabling unsafe based query processing
*/
@CarbonProperty
@@ -1248,17 +1102,6 @@ public final class CarbonCommonConstants {
public static final String ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE = "false";
/**
- * property for offheap based processing
- */
- @CarbonProperty
- public static final String USE_OFFHEAP_IN_QUERY_PROCSSING = "use.offheap.in.query.processing";
-
- /**
- * default value of offheap based processing
- */
- public static final String USE_OFFHEAP_IN_QUERY_PROCSSING_DEFAULT = "true";
-
- /**
* whether to prefetch data while loading.
*/
@CarbonProperty
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index f4566ac..aeaa205 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -231,7 +231,6 @@ public class CarbonUpdateUtil {
// if this call is coming from the delete delta flow then the time stamp
// String will come empty then no need to write into table status file.
if (isTimestampUpdationRequired) {
- loadMetadata.setIsDeleted(CarbonCommonConstants.KEYWORD_TRUE);
// if in case of update flow.
if (loadMetadata.getUpdateDeltaStartTimestamp().isEmpty()) {
// this means for first time it is getting updated .
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index 85602bc..e2bcc8a 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -63,7 +63,6 @@ public class LoadMetadataDetails implements Serializable {
// partition count of this segment
private String partitionCount;
- private String isDeleted = CarbonCommonConstants.KEYWORD_FALSE;
private String dataSize;
private String indexSize;
@@ -339,24 +338,6 @@ public class LoadMetadataDetails implements Serializable {
}
/**
- * To get isDeleted property.
- *
- * @return isDeleted
- */
- public String getIsDeleted() {
- return isDeleted;
- }
-
- /**
- * To set isDeleted property.
- *
- * @param isDeleted
- */
- public void setIsDeleted(String isDeleted) {
- this.isDeleted = isDeleted;
- }
-
- /**
* To get the update delta end timestamp
*
* @return updateDeltaEndTimestamp
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
index 3fed15d..e668de7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/test/Spark2TestQueryExecutor.scala
@@ -40,11 +40,13 @@ class Spark2TestQueryExecutor extends TestQueryExecutorRegister {
}
object Spark2TestQueryExecutor {
+
+ private val STORE_LOCATION_TEMP_PATH = "carbon.tempstore.location"
+
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
LOGGER.info("use TestQueryExecutorImplV2")
CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
- System.getProperty("java.io.tmpdir"))
+ .addProperty(STORE_LOCATION_TEMP_PATH, System.getProperty("java.io.tmpdir"))
.addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FORCE")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0d4de94/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 75fcea3..ac31fcb 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -38,8 +38,6 @@ import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
-import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthEquiSplitGenerator;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.datatype.DataType;
@@ -47,7 +45,6 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.datatypes.GenericDataType;
import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
@@ -371,22 +368,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
LOGGER.info("Number of rows per column blocklet " + pageSize);
dataRows = new ArrayList<>(this.pageSize);
- int dimSet =
- Integer.parseInt(CarbonCommonConstants.DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE);
- // if at least one dimension is present then initialize column splitter otherwise null
- int noOfColStore = colGrpModel.getNoOfColumnStore();
- int[] keyBlockSize = new int[noOfColStore + getExpandedComplexColsCount()];
-
- if (model.getDimLens().length > 0) {
- //Using Variable length variable split generator
- //This will help in splitting mdkey to columns. variable split is required because all
- // columns which are part of
- //row store will be in single column store
- //e.g if {0,1,2,3,4,5} is dimension and {0,1,2) is row store dimension
- //than below splitter will return column as {0,1,2}{3}{4}{5}
- ColumnarSplitter columnarSplitter = model.getSegmentProperties().getFixedLengthKeySplitter();
- System.arraycopy(columnarSplitter.getBlockKeySize(), 0, keyBlockSize, 0, noOfColStore);
- }
// agg type
List<Integer> otherMeasureIndexList =
@@ -411,46 +392,12 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
customMeasureIndex[i] = customMeasureIndexList.get(i);
}
setComplexMapSurrogateIndex(model.getDimensionCount());
- int[] blockKeySize = getBlockKeySizeWithComplexTypes(new MultiDimKeyVarLengthEquiSplitGenerator(
- CarbonUtil.getIncrementedCardinalityFullyFilled(model.getDimLens().clone()), (byte) dimSet)
- .getBlockKeySize());
- System.arraycopy(blockKeySize, noOfColStore, keyBlockSize, noOfColStore,
- blockKeySize.length - noOfColStore);
this.dataWriter = getFactDataWriter();
// initialize the channel;
this.dataWriter.initializeWriter();
}
/**
- * This method combines primitive dimensions with complex metadata columns
- *
- * @param primitiveBlockKeySize
- * @return all dimensions cardinality including complex dimension metadata column
- */
- private int[] getBlockKeySizeWithComplexTypes(int[] primitiveBlockKeySize) {
- int allColsCount = getExpandedComplexColsCount();
- int[] blockKeySizeWithComplexTypes =
- new int[this.colGrpModel.getNoOfColumnStore() + allColsCount];
-
- List<Integer> blockKeySizeWithComplex =
- new ArrayList<Integer>(blockKeySizeWithComplexTypes.length);
- int dictDimensionCount = model.getDimensionCount();
- for (int i = 0; i < dictDimensionCount; i++) {
- GenericDataType complexDataType = model.getComplexIndexMap().get(i);
- if (complexDataType != null) {
- complexDataType.fillBlockKeySize(blockKeySizeWithComplex, primitiveBlockKeySize);
- } else {
- blockKeySizeWithComplex.add(primitiveBlockKeySize[i]);
- }
- }
- for (int i = 0; i < blockKeySizeWithComplexTypes.length; i++) {
- blockKeySizeWithComplexTypes[i] = blockKeySizeWithComplex.get(i);
- }
-
- return blockKeySizeWithComplexTypes;
- }
-
- /**
* Below method will be used to get the fact data writer instance
*
* @return data writer instance