You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/03/30 05:12:15 UTC
[07/13] incubator-carbondata git commit: Removed kettle related code
and refactored
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGeneratorException.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGeneratorException.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGeneratorException.java
deleted file mode 100644
index 6a35994..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGeneratorException.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.graphgenerator;
-
-import java.util.Locale;
-
-public class GraphGeneratorException extends Exception {
-
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * The Error message.
- */
- private String msg = "";
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public GraphGeneratorException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public GraphGeneratorException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
deleted file mode 100644
index f389a44..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
+++ /dev/null
@@ -1,894 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.graphgenerator.configuration;
-
-import java.util.Map;
-
-import org.apache.carbondata.processing.schema.metadata.ColumnSchemaDetailsWrapper;
-import org.apache.carbondata.processing.schema.metadata.TableOptionWrapper;
-
-public class GraphConfigurationInfo {
- private String connectionName;
-
- private String dbType;
-
- private String numberOfCores;
-
- private String storeLocation;
-
- private String tableName;
-
- private String blockletSize;
-
- private String maxBlockletInFile;
-
- private String batchSize;
-
- private Map<String, String> dimCardinalities;
-
- private String[] dimensions;
-
- private String noDictionaryDims;
-
- private String[] measures;
-
- private String dimensionString;
-
- private String hiersString;
-
- private String measuresString;
-
- private String propertiesString;
-
- private String timeHeirString;
-
- private String metaHeirString;
-
- private String metaHeirQueryString;
-
- private String jndiName;
-
- private Map<String, String> tableMeasuresAndDataTypeMap;
-
- private String tableInputSqlQuery;
-
- private String dimensionSqlQuery;
-
- private String sortSize;
-
- private boolean isAGG;
-
- private String driverclass;
-
- private String username;
-
- private String password;
-
- private String connectionUrl;
-
- private String[] actualDims;
-
- /**
- * Sets the dimension:hirearchy#levelnames1,levelName2
- */
- private String dimensionTableNames;
-
- /**
- * column Ids concatenated by a delimeter
- */
- private String dimensionColumnIds;
-
- /**
- * Agg type
- */
- private String[] aggType;
-
- /**
- * mdkeySize
- */
- private String mdkeySize;
-
- /**
- * complexTypeString
- */
- private String complexTypeString;
-
- /**
- * measureCount
- */
-
- private String measureCount;
-
- /**
- * heirAndKeySizeString
- */
- private String heirAndKeySizeString;
-
- /**
- * hier and containing columns string
- */
- private String hierColumnString;
-
- /**
- * forignKey
- */
- private String[] forignKey;
-
- /**
- * Foreign Key and respective hierarchy
- */
- private String foreignKeyHierarchyString;
-
- /**
- * Primary key String
- */
- private String primaryKeyString;
-
- /**
- * Measure Names
- */
- private String measureNamesString;
-
- /**
- * Measure Names
- */
- private String measureUniqueColumnNamesString;
-
- /**
- * actualDimensionColumns
- */
- private String actualDimensionColumns;
-
- /**
- * normHiers
- */
- private String normHiers;
-
- private String forgienKeyAndPrimaryKeyMapString;
-
- /**
- * heirAndDimLens
- */
- private String heirAndDimLens;
-
- /**
- * measureTypeInfo
- */
- private String measureDataTypeInfo;
-
- /**
- * columnAndTableName_ColumnMapForAgg
- */
- private String columnAndTableNameColumnMapForAgg;
- /**
- * denormColumns
- */
- private String denormColumns;
- private String[] aggClass;
- /**
- * type
- */
- private char[] type;
- private String levelAnddataType;
-
- private Boolean[] isNoDictionaryDimMapping;
-
- private Boolean[] isUseInvertedIndex;
-
- private String columnPropertiesString;
-
- /**
- * wrapper object holding the columnschemadetails
- */
- private ColumnSchemaDetailsWrapper columnSchemaDetailsWrapper;
-
- /**
- * wrapper object holding the table options details needed while dataload
- */
- private TableOptionWrapper tableOptionWrapper;
-
- /**
- * It is column groups in below format
- * 0,1~2~3,4,5,6~7~8,9
- * groups are
- * ,-> all ordinal with different group id
- * ~-> all ordinal with same group id
- */
- private String columnGroupsString;
- private String columnsDataTypeString;
- /**
- * @return isUseInvertedIndex
- */
- public Boolean[] getIsUseInvertedIndex() {
- return isUseInvertedIndex;
- }
-
- /**
- * @param isUseInvertedIndex the bool array whether use inverted index to set
- */
- public void setIsUseInvertedIndex(Boolean[] isUseInvertedIndex) {
- this.isUseInvertedIndex = isUseInvertedIndex;
- }
-
- /**
- * @param connectionName the connectionName to set
- */
- public void setConnectionName(String connectionName) {
- this.connectionName = connectionName;
- }
-
- /**
- * @return the numberOfCores
- */
- public String getNumberOfCores() {
- return numberOfCores;
- }
-
- /**
- * @param numberOfCores the numberOfCores to set
- */
- public void setNumberOfCores(String numberOfCores) {
- this.numberOfCores = numberOfCores;
- }
-
- /**
- * @param storeLocation the storeLocation to set
- */
- public void setStoreLocation(String storeLocation) {
- this.storeLocation = storeLocation;
- }
-
- /**
- * @return the tableName
- */
- public String getTableName() {
- return tableName;
- }
-
- /**
- * @param tableName the tableName to set
- */
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public String getComplexTypeString() {
- return complexTypeString;
- }
-
- public void setComplexTypeString(String complexTypeString) {
- this.complexTypeString = complexTypeString;
- }
-
- /**
- * @param blockletSize the blockletSize to set
- */
- public void setBlockletSize(String blockletSize) {
- this.blockletSize = blockletSize;
- }
-
- /**
- * @param maxBlockletInFile the maxBlockletInFile to set
- */
- public void setMaxBlockletInFile(String maxBlockletInFile) {
- this.maxBlockletInFile = maxBlockletInFile;
- }
-
- /**
- * @return the batchSize
- */
- public String getBatchSize() {
- return batchSize;
- }
-
- /**
- * @param batchSize the batchSize to set
- */
- public void setBatchSize(String batchSize) {
- this.batchSize = batchSize;
- }
-
- /**
- * @return the dimCardinalities
- */
- public Map<String, String> getDimCardinalities() {
- return dimCardinalities;
- }
-
- /**
- * @param dimCardinalities the dimCardinalities to set
- */
- public void setDimCardinalities(Map<String, String> dimCardinalities) {
- this.dimCardinalities = dimCardinalities;
- }
-
- /**
- * @return the dimensions
- */
- public String[] getDimensions() {
- return dimensions;
- }
-
- /**
- * @param dimensions the dimensions to set
- */
- public void setDimensions(String[] dimensions) {
- this.dimensions = dimensions;
- }
-
- /**
- * @return the measures
- */
- public String[] getMeasures() {
- return measures;
- }
-
- /**
- * @param measures the measures to set
- */
- public void setMeasures(String[] measures) {
- this.measures = measures;
- }
-
- /**
- * @return the dimensionString
- */
- public String getDimensionString() {
- return dimensionString;
- }
-
- /**
- * @param dimensionString the dimensionString to set
- */
- public void setDimensionString(String dimensionString) {
- this.dimensionString = dimensionString;
- }
-
- /**
- * getNormHiers
- *
- * @return String
- */
- public String getNormHiers() {
- return normHiers;
- }
-
- /**
- * @return the hiersString
- */
- public String getHiersString() {
- return hiersString;
- }
-
- /**
- * @param hiersString the hiersString to set
- */
- public void setHiersString(String hiersString) {
- this.hiersString = hiersString;
- }
-
- /**
- * @return the measuresString
- */
- public String getMeasuresString() {
- return measuresString;
- }
-
- /**
- * @param measuresString the measuresString to set
- */
- public void setMeasuresString(String measuresString) {
- this.measuresString = measuresString;
- }
-
- /**
- * @return the propertiesString
- */
- public String getPropertiesString() {
- return propertiesString;
- }
-
- /**
- * @return the metaHeirString
- */
- public String getMetaHeirString() {
- return metaHeirString;
- }
-
- /**
- * @param metaHeirString the metaHeirString to set
- */
- public void setMetaHeirString(String metaHeirString) {
- this.metaHeirString = metaHeirString;
- }
-
- /**
- * @return the tableInputSqlQuery
- */
- public String getTableInputSqlQuery() {
- return tableInputSqlQuery;
- }
-
- /**
- * @param tableInputSqlQuery the tableInputSqlQuery to set
- */
- public void setTableInputSqlQuery(String tableInputSqlQuery) {
- this.tableInputSqlQuery = tableInputSqlQuery;
- }
-
- /**
- * @return the dimensionSqlQuery
- */
- public String getDimensionSqlQuery() {
- return dimensionSqlQuery;
- }
-
- /**
- * @param dimensionSqlQuery the dimensionSqlQuery to set
- */
- public void setDimensionSqlQuery(String dimensionSqlQuery) {
- this.dimensionSqlQuery = dimensionSqlQuery;
- }
-
- /**
- * @param sortSize the sortSize to set
- */
- public void setSortSize(String sortSize) {
- this.sortSize = sortSize;
- }
-
- /**
- * @return the isAGG
- */
- public boolean isAGG() {
- return isAGG;
- }
-
- /**
- * @param isAGG the isAGG to set
- */
- public void setAGG(boolean isAGG) {
- this.isAGG = isAGG;
- }
-
- /**
- * @return the driverclass
- */
- public String getDriverclass() {
- return driverclass;
- }
-
- /**
- * @param driverclass the driverclass to set
- */
- public void setDriverclass(String driverclass) {
- this.driverclass = driverclass;
- }
-
- /**
- * @return the username
- */
- public String getUsername() {
- return username;
- }
-
- /**
- * @param username the username to set
- */
- public void setUsername(String username) {
- this.username = username;
- }
-
- /**
- * @return the password
- */
- public String getPassword() {
- return password;
- }
-
- /**
- * @param password the password to set
- */
- public void setPassword(String password) {
- this.password = password;
- }
-
- /**
- * @return the connectionUrl
- */
- public String getConnectionUrl() {
- return connectionUrl;
- }
-
- /**
- * @param connectionUrl the connectionUrl to set
- */
- public void setConnectionUrl(String connectionUrl) {
- this.connectionUrl = connectionUrl;
- }
-
- /**
- * @return the actualDims
- */
- public String[] getActualDims() {
- return actualDims;
- }
-
- /**
- * @param actualDims the actualDims to set
- */
- public void setActualDims(String[] actualDims) {
- this.actualDims = actualDims;
- }
-
- /**
- * @return the dimensionTableNames
- */
- public String getDimensionTableNames() {
- return dimensionTableNames;
- }
-
- /**
- * @param dimensionTableNames the dimensionTableNames to set
- */
- public void setDimensionTableNames(String dimensionTableNames) {
- this.dimensionTableNames = dimensionTableNames;
- }
-
- /**
- * @return
- */
- public String getDimensionColumnIds() {
- return dimensionColumnIds;
- }
-
- /**
- * @param dimensionColumnIds column Ids for dimensions in a table
- */
- public void setDimensionColumnIds(String dimensionColumnIds) {
- this.dimensionColumnIds = dimensionColumnIds;
- }
-
- /**
- * getMdkeySize
- *
- * @return String
- */
- public String getMdkeySize() {
- return mdkeySize;
- }
-
- /**
- * setMdkeySize
- *
- * @param mdkeySize void
- */
- public void setMdkeySize(String mdkeySize) {
- this.mdkeySize = mdkeySize;
- }
-
- /**
- * getMeasureCount
- *
- * @return String
- */
- public String getMeasureCount() {
- return measureCount;
- }
-
- /**
- * setMeasureCount
- *
- * @param measureCount void
- */
- public void setMeasureCount(String measureCount) {
- this.measureCount = measureCount;
- }
-
- /**
- * getHeirAndKeySizeString
- *
- * @return String
- */
- public String getHeirAndKeySizeString() {
- return heirAndKeySizeString;
- }
-
- /**
- * setHeirAndKeySizeString
- *
- * @param heirAndKeySizeString void
- */
- public void setHeirAndKeySizeString(String heirAndKeySizeString) {
- this.heirAndKeySizeString = heirAndKeySizeString;
- }
-
- /**
- * @return Returns the hierColumnString.
- */
- public String getHierColumnString() {
- return hierColumnString;
- }
-
- /**
- * @param hierColumnString The hierColumnString to set.
- */
- public void setHierColumnString(String hierColumnString) {
- this.hierColumnString = hierColumnString;
- }
-
- /**
- * @param forignKey The forignKey to set.
- */
- public void setForignKey(String[] forignKey) {
- this.forignKey = forignKey;
- }
-
- /**
- * @return Returns the foreignKeyHierarchyString.
- */
- public String getForeignKeyHierarchyString() {
- return foreignKeyHierarchyString;
- }
-
- /**
- * @param foreignKeyHierarchyString The foreignKeyHierarchyString to set.
- */
- public void setForeignKeyHierarchyString(String foreignKeyHierarchyString) {
- this.foreignKeyHierarchyString = foreignKeyHierarchyString;
- }
-
- /**
- * @return Returns the primaryKeyString.
- */
- public String getPrimaryKeyString() {
- return primaryKeyString;
- }
-
- /**
- * @param primaryKeyString The primaryKeyString to set.
- */
- public void setPrimaryKeyString(String primaryKeyString) {
- this.primaryKeyString = primaryKeyString;
- }
-
- /**
- * @return the measureNamesString
- */
- public String getMeasureNamesString() {
- return measureNamesString;
- }
-
- /**
- * @param measureNamesString the measureNamesString to set
- */
- public void setMeasureNamesString(String measureNamesString) {
- this.measureNamesString = measureNamesString;
- }
-
- /**
- * @return Returns the aggType.
- */
- public String[] getAggType() {
- return aggType;
- }
-
- /**
- * @param aggType The aggType to set.
- */
- public void setAggType(String[] aggType) {
- this.aggType = aggType;
- }
-
- /**
- * @return Returns the actualDimensionColumns.
- */
- public String getActualDimensionColumns() {
- return actualDimensionColumns;
- }
-
- /**
- * @param actualDimensionColumns The actualDimensionColumns to set.
- */
- public void setActualDimensionColumns(String actualDimensionColumns) {
- this.actualDimensionColumns = actualDimensionColumns;
- }
-
- /**
- * getForgienKeyAndPrimaryKeyMapString
- *
- * @return String
- */
- public String getForgienKeyAndPrimaryKeyMapString() {
- return forgienKeyAndPrimaryKeyMapString;
- }
-
- /**
- * setForgienKeyAndPrimaryKeyMapString
- *
- * @param forgienKeyAndPrimaryKeyMapString void
- */
- public void setForgienKeyAndPrimaryKeyMapString(String forgienKeyAndPrimaryKeyMapString) {
- this.forgienKeyAndPrimaryKeyMapString = forgienKeyAndPrimaryKeyMapString;
- }
-
- /**
- * @return Returns the heirAndDimLens.
- */
- public String getHeirAndDimLens() {
- return heirAndDimLens;
- }
-
- /**
- * @param heirAndDimLens The heirAndDimLens to set.
- */
- public void setHeirAndDimLens(String heirAndDimLens) {
- this.heirAndDimLens = heirAndDimLens;
- }
-
- /**
- * @return Returns the measureDataTypeInfo.
- */
- public String getMeasureDataTypeInfo() {
- return measureDataTypeInfo;
- }
-
- /**
- * @param measureDataTypeInfo The measureDataTypeInfo to set.
- */
- public void setMeasureDataTypeInfo(String measureDataTypeInfo) {
- this.measureDataTypeInfo = measureDataTypeInfo;
- }
-
- public String getColumnAndTableNameColumnMapForAgg() {
- return columnAndTableNameColumnMapForAgg;
- }
-
- /**
- * @return Returns the denormColumns.
- */
- public String getDenormColumns() {
- return denormColumns;
- }
-
- /**
- * @param denormColumns The denormColumns to set.
- */
- public void setDenormColumns(String denormColumns) {
- this.denormColumns = denormColumns;
- }
-
- /**
- * @return the aggClass
- */
- public String[] getAggClass() {
- return aggClass;
- }
-
- /**
- * @param measureUniqueColumnNamesString the measureUniqueColumnNamesString to set
- */
- public void setMeasureUniqueColumnNamesString(String measureUniqueColumnNamesString) {
- this.measureUniqueColumnNamesString = measureUniqueColumnNamesString;
- }
-
- /**
- * @param type the type to set
- */
- public void setType(char[] type) {
- this.type = type;
- }
-
- public String getLevelAnddataType() {
- return levelAnddataType;
- }
-
- public void setLevelAnddataType(String levelAnddataType) {
- this.levelAnddataType = levelAnddataType;
- }
-
- /**
- * getNoDictionaryDims.
- *
- * @return
- */
- public String getNoDictionaryDims() {
- return noDictionaryDims;
- }
-
- /**
- * setNoDictionaryDims.
- *
- * @param noDictionaryDims
- */
- public void setNoDictionaryDims(String noDictionaryDims) {
- this.noDictionaryDims = noDictionaryDims;
- }
-
- /**
- * Set Wrapper Object having the columnschemadetails
- *
- * @param columnSchemaDetailsWrapper
- */
- public void setColumnSchemaDetails(ColumnSchemaDetailsWrapper columnSchemaDetailsWrapper) {
- this.columnSchemaDetailsWrapper = columnSchemaDetailsWrapper;
- }
-
- /**
- * return the Wrapper Object having the columnschemadetails
- * @return
- */
- public ColumnSchemaDetailsWrapper getColumnSchemaDetails() {
- return columnSchemaDetailsWrapper;
- }
-
- /**
- * set wraper object having table options needed while dataload
- *
- * @return
- */
- public void setTableOptionWrapper(TableOptionWrapper tableOptionWrapper) {
- this.tableOptionWrapper = tableOptionWrapper;
- }
-
- /**
- * method returns the table options detail wrapper instance.
- * @return
- */
- public TableOptionWrapper getTableOptionWrapper() {
- return tableOptionWrapper;
- }
-
- public void setColumnGroupsString(String columnGroups) {
- this.columnGroupsString = columnGroups;
- }
-
- /**
- * @return columngroups
- */
- public String getColumnGroupsString() {
- return columnGroupsString;
- }
-
- public Boolean[] getIsNoDictionaryDimMapping() {
- return isNoDictionaryDimMapping;
- }
-
- public void setIsNoDictionaryDimMapping(Boolean[] isNoDictionaryDimMapping) {
- this.isNoDictionaryDimMapping = isNoDictionaryDimMapping;
- }
-
- public void setColumnPropertiesString(String columnPropertiesString) {
- this.columnPropertiesString = columnPropertiesString;
- }
-
- public String getColumnPropertiesString() {
- return this.columnPropertiesString;
- }
-
- /**
- * @return columngroups
- */
- public String getDimensionColumnsDataType() {
- return columnsDataTypeString;
- }
-
- public void setDimensionColumnsDataType(String columnsDataTypeString) {
- this.columnsDataTypeString = columnsDataTypeString;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
deleted file mode 100644
index 2f92699..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
+++ /dev/null
@@ -1,531 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.mdkeygen;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.common.logging.impl.StandardLogService;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.processing.datatypes.GenericDataType;
-import org.apache.carbondata.processing.mdkeygen.file.FileData;
-import org.apache.carbondata.processing.mdkeygen.file.FileManager;
-import org.apache.carbondata.processing.mdkeygen.file.IFileManagerComposite;
-import org.apache.carbondata.processing.store.CarbonDataFileAttributes;
-import org.apache.carbondata.processing.store.CarbonFactDataHandlerColumnar;
-import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
-import org.apache.carbondata.processing.store.CarbonFactHandler;
-import org.apache.carbondata.processing.store.SingleThreadFinalSortFilesMerger;
-import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
-import org.apache.carbondata.processing.util.NonDictionaryUtil;
-
-import org.pentaho.di.core.exception.KettleException;
-import org.pentaho.di.core.row.RowMeta;
-import org.pentaho.di.core.row.ValueMeta;
-import org.pentaho.di.core.row.ValueMetaInterface;
-import org.pentaho.di.trans.Trans;
-import org.pentaho.di.trans.TransMeta;
-import org.pentaho.di.trans.step.BaseStep;
-import org.pentaho.di.trans.step.StepDataInterface;
-import org.pentaho.di.trans.step.StepMeta;
-import org.pentaho.di.trans.step.StepMetaInterface;
-
-public class MDKeyGenStep extends BaseStep {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(MDKeyGenStep.class.getName());
-
- /**
- * carbon mdkey generator step data class
- */
- private MDKeyGenStepData data;
-
- /**
- * carbon mdkey generator step meta
- */
- private MDKeyGenStepMeta meta;
-
- /**
- * dimension length
- */
- private int dimensionCount;
-
- /**
- * table name
- */
- private String tableName;
-
- /**
- * table blocksize in MB
- */
- private int blockSize;
-
- /**
- * File manager
- */
- private IFileManagerComposite fileManager;
-
- private Map<Integer, GenericDataType> complexIndexMap;
-
- private SegmentProperties segmentProperties;
-
- private int[] colCardinality;
-
- private List<ColumnSchema> wrapperColumnSchema;
-
- /**
- * readCounter
- */
- private long readCounter;
-
- /**
- * writeCounter
- */
- private long writeCounter;
-
- private int measureCount;
-
- private String dataFolderLocation;
-
- private SingleThreadFinalSortFilesMerger finalMerger;
-
- /**
- * dataHandler
- */
- private CarbonFactHandler dataHandler;
-
- private char[] aggType;
-
- private String storeLocation;
-
- private int[] dimLens;
-
- /**
- * to check whether dimension is of dictionary type
- * or not
- */
- private boolean[] isNoDictionaryDimension;
-
- /**
- * to check whether dimension use inverted index
- * or not
- */
- private boolean[] isUseInvertedIndex;
-
- /**
- * CarbonMDKeyGenStep
- *
- * @param stepMeta
- * @param stepDataInterface
- * @param copyNr
- * @param transMeta
- * @param trans
- */
- public MDKeyGenStep(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr,
- TransMeta transMeta, Trans trans) {
- super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
- }
-
- /**
- * Perform the equivalent of processing one row. Typically this means
- * reading a row from input (getRow()) and passing a row to output
- * (putRow)).
- *
- * @param smi The steps metadata to work with
- * @param sdi The steps temporary working data to work with (database
- * connections, result sets, caches, temporary variables, etc.)
- * @return false if no more rows can be processed or an error occurred.
- * @throws KettleException
- */
- public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
- meta = (MDKeyGenStepMeta) smi;
- StandardLogService.setThreadName(meta.getPartitionID(), null);
- data = (MDKeyGenStepData) sdi;
-
- meta.initialize();
- Object[] row = getRow();
- if (first) {
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordMdkGenerateTotalTime(
- meta.getPartitionID(), System.currentTimeMillis());
- first = false;
-
- data.outputRowMeta = new RowMeta();
- boolean isExecutionRequired = setStepConfiguration();
-
- if (!isExecutionRequired) {
- processingComplete();
- return false;
- }
- setStepOutputInterface();
- }
-
- if (null != row) {
- putRow(data.outputRowMeta, new Object[measureCount + 1]);
- return true;
- }
-
- try {
- initDataHandler();
- dataHandler.initialise();
- finalMerger.startFinalMerge();
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(
- meta.getPartitionID(), System.currentTimeMillis());
- while (finalMerger.hasNext()) {
- Object[] r = finalMerger.next();
- readCounter++;
- Object[] outputRow = process(r);
- dataHandler.addDataToStore(outputRow);
- writeCounter++;
- }
- } catch (CarbonDataWriterException e) {
- LOGGER.error(e, "Failed for table: " + this.tableName + " in MDKeyGenStep");
- throw new KettleException("Error while initializing data handler : " + e.getMessage());
- } catch (Exception e) {
- LOGGER.error(e, "Failed for table: " + this.tableName + " in MDKeyGenStep");
- throw new KettleException("There is an unexpected error: " + e.getMessage());
- } finally {
- try {
- dataHandler.finish();
- } catch (CarbonDataWriterException e) {
- LOGGER.error(e, "Failed for table: " + this.tableName + " in finishing data handler");
- } catch (Exception e) {
- LOGGER.error(e, "Failed for table: " + this.tableName + " in finishing data handler");
- }
- }
- LOGGER.info("Record Procerssed For table: " + this.tableName);
- String logMessage =
- "Finished Carbon Mdkey Generation Step: Read: " + readCounter + ": Write: " + writeCounter;
- LOGGER.info(logMessage);
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(writeCounter);
- processingComplete();
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(
- meta.getPartitionID(), System.currentTimeMillis());
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordMdkGenerateTotalTime(
- meta.getPartitionID(), System.currentTimeMillis());
- return false;
- }
-
- private void processingComplete() throws KettleException {
- if (null != dataHandler) {
- try {
- dataHandler.closeHandler();
- } catch (CarbonDataWriterException e) {
- LOGGER.error(e, e.getMessage());
- throw new KettleException(e.getMessage());
- } catch (Exception e) {
- LOGGER.error(e, e.getMessage());
- throw new KettleException("There is an unexpected error: " + e.getMessage());
- }
- }
- setOutputDone();
- }
-
- /**
- * This method will be used to get and update the step properties which will
- * required to run this step
- */
- private boolean setStepConfiguration() {
- this.tableName = meta.getTableName();
- storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(meta.getDatabaseName(), meta.getTableName(),
- String.valueOf(meta.getTaskNo()), meta.getPartitionID(), meta.getSegmentId() + "",
- false);
- isNoDictionaryDimension =
- NonDictionaryUtil.convertStringToBooleanArr(meta.getNoDictionaryDimsMapping());
- isUseInvertedIndex =
- NonDictionaryUtil.convertStringToBooleanArr(meta.getIsUseInvertedIndex());
- fileManager = new FileManager();
- fileManager.setName(CarbonCommonConstants.LOAD_FOLDER + meta.getSegmentId()
- + CarbonCommonConstants.FILE_INPROGRESS_STATUS);
-
- if (!(new File(storeLocation).exists())) {
- LOGGER.error("Local data load folder location does not exist: " + storeLocation);
- return false;
- }
-
- this.meta.setNoDictionaryCount(
- NonDictionaryUtil.extractNoDictionaryCount(this.meta.getNoDictionaryDims()));
-
- String levelCardinalityFilePath = storeLocation + File.separator +
- CarbonCommonConstants.LEVEL_METADATA_FILE + meta.getTableName()
- + CarbonCommonConstants.CARBON_METADATA_EXTENSION;
- int[] dimLensWithComplex = null;
- try {
- dimLensWithComplex = CarbonUtil.getCardinalityFromLevelMetadataFile(levelCardinalityFilePath);
- } catch (IOException e) {
- LOGGER.error("Level cardinality file :: " + e.getMessage());
- return false;
- }
- if (null == dimLensWithComplex) {
- return false;
- }
- List<Integer> dimsLenList = new ArrayList<Integer>();
- for (int eachDimLen : dimLensWithComplex) {
- if (eachDimLen != 0) dimsLenList.add(eachDimLen);
- }
- dimLens = new int[dimsLenList.size()];
- for (int i = 0; i < dimsLenList.size(); i++) {
- dimLens[i] = dimsLenList.get(i);
- }
-
- this.dimensionCount = meta.getDimensionCount();
-
- int simpleDimsCount = this.dimensionCount - meta.getComplexDimsCount();
- int[] simpleDimsLen = new int[simpleDimsCount];
- System.arraycopy(dimLens, 0, simpleDimsLen, 0, simpleDimsCount);
-
- CarbonTable carbonTable = CarbonMetadata.getInstance()
- .getCarbonTable(meta.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + tableName);
- wrapperColumnSchema = CarbonUtil
- .getColumnSchemaList(carbonTable.getDimensionByTableName(tableName),
- carbonTable.getMeasureByTableName(tableName));
- blockSize = carbonTable.getBlockSizeInMB();
- colCardinality =
- CarbonUtil.getFormattedCardinality(dimLensWithComplex, wrapperColumnSchema);
- segmentProperties = new SegmentProperties(wrapperColumnSchema, colCardinality);
- // Actual primitive dimension used to generate start & end key
-
- data.generator = segmentProperties.getDimensionKeyGenerator();
-
- //To Set MDKey Index of each primitive type in complex type
- int surrIndex = simpleDimsCount;
- Iterator<Entry<String, GenericDataType>> complexMap =
- meta.getComplexTypes().entrySet().iterator();
- complexIndexMap = new HashMap<Integer, GenericDataType>(meta.getComplexDimsCount());
- while (complexMap.hasNext()) {
- Entry<String, GenericDataType> complexDataType = complexMap.next();
- complexDataType.getValue().setOutputArrayIndex(0);
- complexIndexMap.put(simpleDimsCount, complexDataType.getValue());
- simpleDimsCount++;
- List<GenericDataType> primitiveTypes = new ArrayList<GenericDataType>();
- complexDataType.getValue().getAllPrimitiveChildren(primitiveTypes);
- for (GenericDataType eachPrimitive : primitiveTypes) {
- eachPrimitive.setSurrogateIndex(surrIndex++);
- }
- }
-
- this.measureCount = meta.getMeasureCount();
-
- String metaDataFileName = CarbonCommonConstants.MEASURE_METADATA_FILE_NAME + this.tableName
- + CarbonCommonConstants.MEASUREMETADATA_FILE_EXT
- + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
-
- FileData fileData = new FileData(metaDataFileName, storeLocation);
- fileManager.add(fileData);
- // Set the data file location
- this.dataFolderLocation =
- storeLocation + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION;
- return true;
- }
-
- private void initDataHandler() {
- int simpleDimsCount = this.dimensionCount - meta.getComplexDimsCount();
- int[] simpleDimsLen = new int[simpleDimsCount];
- System.arraycopy(dimLens, 0, simpleDimsLen, 0, simpleDimsCount);
- CarbonDataFileAttributes carbonDataFileAttributes =
- new CarbonDataFileAttributes(meta.getTaskNo(), meta.getFactTimeStamp());
- initAggType();
- String carbonDataDirectoryPath = getCarbonDataFolderLocation();
- finalMerger = new SingleThreadFinalSortFilesMerger(dataFolderLocation, tableName,
- dimensionCount - meta.getComplexDimsCount(), meta.getComplexDimsCount(), measureCount,
- meta.getNoDictionaryCount(), aggType, isNoDictionaryDimension, true);
- CarbonFactDataHandlerModel carbonFactDataHandlerModel = getCarbonFactDataHandlerModel();
- carbonFactDataHandlerModel.setPrimitiveDimLens(simpleDimsLen);
- carbonFactDataHandlerModel.setCarbonDataFileAttributes(carbonDataFileAttributes);
- carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
- carbonFactDataHandlerModel.setIsUseInvertedIndex(isUseInvertedIndex);
- if (meta.getNoDictionaryCount() > 0 || meta.getComplexDimsCount() > 0) {
- carbonFactDataHandlerModel.setMdKeyIndex(measureCount + 1);
- } else {
- carbonFactDataHandlerModel.setMdKeyIndex(measureCount);
- }
- dataHandler = new CarbonFactDataHandlerColumnar(carbonFactDataHandlerModel);
- }
-
- /**
- * This method will create a model object for carbon fact data handler
- *
- * @return
- */
- private CarbonFactDataHandlerModel getCarbonFactDataHandlerModel() {
- CarbonFactDataHandlerModel carbonFactDataHandlerModel = new CarbonFactDataHandlerModel();
- carbonFactDataHandlerModel.setDatabaseName(meta.getDatabaseName());
- carbonFactDataHandlerModel.setTableName(tableName);
- carbonFactDataHandlerModel.setMeasureCount(measureCount);
- carbonFactDataHandlerModel.setMdKeyLength(data.generator.getKeySizeInBytes());
- carbonFactDataHandlerModel.setStoreLocation(storeLocation);
- carbonFactDataHandlerModel.setDimLens(dimLens);
- carbonFactDataHandlerModel.setNoDictionaryCount(meta.getNoDictionaryCount());
- carbonFactDataHandlerModel.setDimensionCount(dimensionCount);
- carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap);
- carbonFactDataHandlerModel.setSegmentProperties(segmentProperties);
- carbonFactDataHandlerModel.setColCardinality(colCardinality);
- carbonFactDataHandlerModel.setDataWritingRequest(true);
- carbonFactDataHandlerModel.setAggType(aggType);
- carbonFactDataHandlerModel.setFactDimLens(dimLens);
- carbonFactDataHandlerModel.setWrapperColumnSchema(wrapperColumnSchema);
- carbonFactDataHandlerModel.setBlockSizeInMB(blockSize);
- return carbonFactDataHandlerModel;
- }
-
- private void initAggType() {
- aggType = new char[measureCount];
- Arrays.fill(aggType, 'n');
- CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(
- meta.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + meta.getTableName());
- List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(meta.getTableName());
- for (int i = 0; i < measureCount; i++) {
- aggType[i] = DataTypeUtil.getAggType(measures.get(i).getDataType());
- }
- }
-
- /**
- * This method will be used for setting the output interface.
- * Output interface is how this step will process the row to next step
- */
- private void setStepOutputInterface() {
- ValueMetaInterface[] out = new ValueMetaInterface[measureCount + 1];
-
- for (int i = 0; i < measureCount; i++) {
- out[i] = new ValueMeta("measure" + i, ValueMetaInterface.TYPE_NUMBER,
- ValueMetaInterface.STORAGE_TYPE_NORMAL);
- out[i].setStorageMetadata(new ValueMeta("measure" + i, ValueMetaInterface.TYPE_NUMBER,
- ValueMetaInterface.STORAGE_TYPE_NORMAL));
- }
-
- out[out.length - 1] = new ValueMeta("id", ValueMetaInterface.TYPE_BINARY,
- ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
- out[out.length - 1].setStorageMetadata(new ValueMeta("id", ValueMetaInterface.TYPE_STRING,
- ValueMetaInterface.STORAGE_TYPE_NORMAL));
- out[out.length - 1].setLength(256);
- out[out.length - 1].setStringEncoding(CarbonCommonConstants.BYTE_ENCODING);
- out[out.length - 1].getStorageMetadata().setStringEncoding(CarbonCommonConstants.BYTE_ENCODING);
-
- data.outputRowMeta.setValueMetaList(Arrays.asList(out));
- }
-
- /**
- * This method will be used to get the row from previous step and then it
- * will generate the mdkey and then send the mdkey to next step
- *
- * @param row input row
- * @throws KettleException
- */
- private Object[] process(Object[] row) throws KettleException {
- Object[] outputRow = null;
- // adding one for the high cardinality dims byte array.
- if (meta.getNoDictionaryCount() > 0 || meta.getComplexDimsCount() > 0) {
- outputRow = new Object[measureCount + 1 + 1];
- } else {
- outputRow = new Object[measureCount + 1];
- }
-
- int l = 0;
- int index = 0;
- for (int i = 0; i < measureCount; i++) {
- if (aggType[i] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- outputRow[l++] = NonDictionaryUtil.getMeasure(index++, row);
- } else if (aggType[i] == CarbonCommonConstants.BIG_INT_MEASURE) {
- outputRow[l++] = (Long) NonDictionaryUtil.getMeasure(index++, row);
- } else {
- outputRow[l++] = (Double) NonDictionaryUtil.getMeasure(index++, row);
- }
- }
- outputRow[l] = NonDictionaryUtil.getByteArrayForNoDictionaryCols(row);
-
- int[] highCardExcludedRows = new int[segmentProperties.getDimColumnsCardinality().length];
- for (int i = 0; i < highCardExcludedRows.length; i++) {
- Object key = NonDictionaryUtil.getDimension(i, row);
- highCardExcludedRows[i] = (Integer) key;
- }
- try {
- outputRow[outputRow.length - 1] = data.generator.generateKey(highCardExcludedRows);
- } catch (KeyGenException e) {
- throw new KettleException("unable to generate the mdkey", e);
- }
-
- return outputRow;
- }
-
- /**
- * This method will get the store location for the given path, segment id and partition id
- *
- * @return data directory path
- */
- private String getCarbonDataFolderLocation() {
- String carbonStorePath =
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS);
- CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(
- meta.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + meta.getTableName());
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTable.getCarbonTableIdentifier());
- String carbonDataDirectoryPath =
- carbonTablePath.getCarbonDataDirectoryPath(meta.getPartitionID(), meta.getSegmentId() + "");
- return carbonDataDirectoryPath;
- }
-
- /**
- * Initialize and do work where other steps need to wait for...
- *
- * @param smi The metadata to work with
- * @param sdi The data to initialize
- * @return step initialize or not
- */
- public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
- meta = (MDKeyGenStepMeta) smi;
- data = (MDKeyGenStepData) sdi;
-
- return super.init(smi, sdi);
- }
-
- /**
- * Dispose of this step: close files, empty logs, etc.
- *
- * @param smi The metadata to work with
- * @param sdi The data to dispose of
- */
- public void dispose(StepMetaInterface smi, StepDataInterface sdi) {
- meta = (MDKeyGenStepMeta) smi;
- data = (MDKeyGenStepData) sdi;
- super.dispose(smi, sdi);
- dataHandler = null;
- finalMerger = null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepData.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepData.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepData.java
deleted file mode 100644
index b2c682c..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepData.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.mdkeygen;
-
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-
-import org.pentaho.di.core.row.RowMetaInterface;
-import org.pentaho.di.trans.step.BaseStepData;
-import org.pentaho.di.trans.step.StepDataInterface;
-
-public class MDKeyGenStepData extends BaseStepData implements StepDataInterface {
- /**
- * outputRowMeta
- */
- protected RowMetaInterface outputRowMeta;
-
- /**
- * generator for each column independently
- */
- protected KeyGenerator generator;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
deleted file mode 100644
index c1be2ef..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.mdkeygen;
-
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.processing.datatypes.ArrayDataType;
-import org.apache.carbondata.processing.datatypes.GenericDataType;
-import org.apache.carbondata.processing.datatypes.PrimitiveDataType;
-import org.apache.carbondata.processing.datatypes.StructDataType;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
-
-import org.pentaho.di.core.CheckResultInterface;
-import org.pentaho.di.core.Counter;
-import org.pentaho.di.core.database.DatabaseMeta;
-import org.pentaho.di.core.exception.KettleException;
-import org.pentaho.di.core.exception.KettleXMLException;
-import org.pentaho.di.core.row.RowMetaInterface;
-import org.pentaho.di.core.xml.XMLHandler;
-import org.pentaho.di.i18n.BaseMessages;
-import org.pentaho.di.repository.ObjectId;
-import org.pentaho.di.repository.Repository;
-import org.pentaho.di.trans.Trans;
-import org.pentaho.di.trans.TransMeta;
-import org.pentaho.di.trans.step.BaseStepMeta;
-import org.pentaho.di.trans.step.StepDataInterface;
-import org.pentaho.di.trans.step.StepInterface;
-import org.pentaho.di.trans.step.StepMeta;
-import org.pentaho.di.trans.step.StepMetaInterface;
-import org.w3c.dom.Node;
-
-public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface {
- /**
- * for i18n purposes
- */
- private static Class<?> pkg = MDKeyGenStepMeta.class;
-
- /**
- * tableName
- */
- private String tableName;
-
- /**
- * numberOfCores
- */
- private String numberOfCores;
-
- /**
- * databaseName
- */
- private String databaseName;
-
- /**
- * aggregateLevels
- */
- private String aggregateLevels;
-
- /**
- * measureCount
- */
- private String measureCount;
-
- /**
- * dimensionCount
- */
- private String dimensionCount;
-
- /**
- * complexDimsCount
- */
- private String complexDimsCount;
-
- /**
- * ComplexTypeString
- */
- private String complexTypeString;
-
- private Map<String, GenericDataType> complexTypes;
-
- /**
- * It is column groups in below format
- * 0,1~2~3,4,5,6~7~8,9
- * groups are
- * ,-> all ordinal with different group id
- * ~-> all ordinal with same group id
- */
- private String columnGroupsString;
- private String noDictionaryDims;
-
- /**
- * noDictionaryCount
- */
- private int noDictionaryCount;
-
- private String measureDataType;
- /**
- * task id, each spark task has a unique id
- */
- private String taskNo;
- /**
- * new load start time
- */
- private String factTimeStamp;
- /**
- * partitionID
- */
- private String partitionID;
- /**
- * Id of the load folder
- */
- private String segmentId;
- /**
- * To determine the column whether is dictionary or not.
- */
- private String noDictionaryDimsMapping;
- /**
- * To determine the column whether use inverted index or not.
- */
- private String isUseInvertedIndex;
-
- /**
- * Constructor
- */
- public MDKeyGenStepMeta() {
- super();
- }
-
- @Override public void setDefault() {
- tableName = "";
- numberOfCores = "";
- aggregateLevels = "";
- tableName = "";
- databaseName = "";
- columnGroupsString = "";
- noDictionaryDims = "";
- measureDataType = "";
- taskNo = "";
- factTimeStamp = "";
- partitionID = "";
- segmentId = "";
- noDictionaryDimsMapping = "";
- }
-
- public String getXML() {
- StringBuilder retval = new StringBuilder(150);
-
- retval.append(" ").append(XMLHandler.addTagValue("TableName", tableName));
- retval.append(" ").append(XMLHandler.addTagValue("AggregateLevels", aggregateLevels));
- retval.append(" ").append(XMLHandler.addTagValue("NumberOfCores", numberOfCores));
- retval.append(" ").append(XMLHandler.addTagValue("tableName", tableName));
- retval.append(" ").append(XMLHandler.addTagValue("databaseName", databaseName));
- retval.append(" ").append(XMLHandler.addTagValue("noDictionaryDims", noDictionaryDims));
- retval.append(" ").append(XMLHandler.addTagValue("measureCount", measureCount));
- retval.append(" ").append(XMLHandler.addTagValue("dimensionsStoreType", columnGroupsString));
- retval.append(" ").append(XMLHandler.addTagValue("dimensionCount", dimensionCount));
- retval.append(" ").append(XMLHandler.addTagValue("complexDimsCount", complexDimsCount));
- retval.append(" ").append(XMLHandler.addTagValue("complexTypeString", complexTypeString));
- retval.append(" ").append(XMLHandler.addTagValue("measureDataType", measureDataType));
- retval.append(" ").append(XMLHandler.addTagValue("taskNo", taskNo));
- retval.append(" ").append(XMLHandler.addTagValue("factTimeStamp", factTimeStamp));
- retval.append(" ").append(XMLHandler.addTagValue("factTimeStamp", factTimeStamp));
- retval.append(" ").append(XMLHandler.addTagValue("partitionID", partitionID));
- retval.append(" ").append(XMLHandler.addTagValue("isUseInvertedIndex", isUseInvertedIndex));
- retval.append(" ").append(XMLHandler.addTagValue("segmentId", segmentId));
- retval.append(" ")
- .append(XMLHandler.addTagValue("noDictionaryDimsMapping", noDictionaryDimsMapping));
- return retval.toString();
- }
-
- @Override
- public void loadXML(Node stepnode, List<DatabaseMeta> databases, Map<String, Counter> counters)
- throws KettleXMLException {
- try {
- tableName = XMLHandler.getTagValue(stepnode, "TableName");
- aggregateLevels = XMLHandler.getTagValue(stepnode, "AggregateLevels");
- numberOfCores = XMLHandler.getTagValue(stepnode, "NumberOfCores");
- databaseName = XMLHandler.getTagValue(stepnode, "databaseName");
- tableName = XMLHandler.getTagValue(stepnode, "tableName");
- noDictionaryDims = XMLHandler.getTagValue(stepnode, "noDictionaryDims");
- measureCount = XMLHandler.getTagValue(stepnode, "measureCount");
- columnGroupsString = XMLHandler.getTagValue(stepnode, "dimensionsStoreType");
- dimensionCount = XMLHandler.getTagValue(stepnode, "dimensionCount");
- complexDimsCount = XMLHandler.getTagValue(stepnode, "complexDimsCount");
- complexTypeString = XMLHandler.getTagValue(stepnode, "complexTypeString");
- measureDataType = XMLHandler.getTagValue(stepnode, "measureDataType");
- taskNo = XMLHandler.getTagValue(stepnode, "taskNo");
- factTimeStamp = XMLHandler.getTagValue(stepnode, "factTimeStamp");
- partitionID = XMLHandler.getTagValue(stepnode, "partitionID");
- isUseInvertedIndex = XMLHandler.getTagValue(stepnode, "isUseInvertedIndex");
- segmentId = XMLHandler.getTagValue(stepnode, "segmentId");
- noDictionaryDimsMapping = XMLHandler.getTagValue(stepnode, "noDictionaryDimsMapping");
- } catch (Exception e) {
- throw new KettleXMLException("Unable to read step info from XML node", e);
- }
- }
-
- @Override public void saveRep(Repository rep, ObjectId idTransformation, ObjectId idStep)
- throws KettleException {
- try {
- rep.saveStepAttribute(idTransformation, idStep, "TableName", tableName);
- rep.saveStepAttribute(idTransformation, idStep, "AggregateLevels", aggregateLevels);
- rep.saveStepAttribute(idTransformation, idStep, "NumberOfCores", numberOfCores);
- rep.saveStepAttribute(idTransformation, idStep, "databaseName", databaseName);
- rep.saveStepAttribute(idTransformation, idStep, "tableName", tableName);
- rep.saveStepAttribute(idTransformation, idStep, "noDictionaryDims", noDictionaryDims);
- rep.saveStepAttribute(idTransformation, idStep, "measureCount", measureCount);
- rep.saveStepAttribute(idTransformation, idStep, "dimensionsStoreType", columnGroupsString);
- rep.saveStepAttribute(idTransformation, idStep, "dimensionCount", dimensionCount);
- rep.saveStepAttribute(idTransformation, idStep, "complexDimsCount", complexDimsCount);
- rep.saveStepAttribute(idTransformation, idStep, "complexTypeString", complexTypeString);
- rep.saveStepAttribute(idTransformation, idStep, "measureDataType", measureDataType);
- rep.saveStepAttribute(idTransformation, idStep, "taskNo", taskNo);
- rep.saveStepAttribute(idTransformation, idStep, "factTimeStamp", factTimeStamp);
- rep.saveStepAttribute(idTransformation, idStep, "partitionID", partitionID);
- rep.saveStepAttribute(idTransformation, idStep, "isUseInvertedIndex", isUseInvertedIndex);
- rep.saveStepAttribute(idTransformation, idStep, "segmentId", segmentId);
- rep.saveStepAttribute(idTransformation, idStep, "noDictionaryDimsMapping",
- noDictionaryDimsMapping);
- } catch (Exception e) {
- throw new KettleException(
- BaseMessages.getString(pkg, "TemplateStep.Exception.UnableToSaveStepInfoToRepository")
- + idStep, e);
- }
-
- }
-
- @Override public void readRep(Repository rep, ObjectId idStep, List<DatabaseMeta> databases,
- Map<String, Counter> counters) throws KettleException {
- try {
- tableName = rep.getStepAttributeString(idStep, "TableName");
- aggregateLevels = rep.getStepAttributeString(idStep, "AggregateLevels");
- numberOfCores = rep.getStepAttributeString(idStep, "NumberOfCores");
- databaseName = rep.getStepAttributeString(idStep, "databaseName");
- tableName = rep.getStepAttributeString(idStep, "tableName");
- noDictionaryDims = rep.getStepAttributeString(idStep, "noDictionaryDims");
- measureCount = rep.getStepAttributeString(idStep, "measureCount");
- columnGroupsString = rep.getStepAttributeString(idStep, "dimensionsStoreType");
- dimensionCount = rep.getStepAttributeString(idStep, "dimensionCount");
- complexDimsCount = rep.getStepAttributeString(idStep, "complexDimsCount");
- complexTypeString = rep.getStepAttributeString(idStep, "complexTypeString");
- measureDataType = rep.getStepAttributeString(idStep, "measureDataType");
- taskNo = rep.getStepAttributeString(idStep, "taskNo");
- factTimeStamp = rep.getStepAttributeString(idStep, "factTimeStamp");
- partitionID = rep.getStepAttributeString(idStep, "partitionID");
- isUseInvertedIndex = rep.getStepAttributeString(idStep, "isUseInvertedIndex");
- segmentId = rep.getStepAttributeString(idStep, "segmentId");
- noDictionaryDimsMapping = rep.getStepAttributeString(idStep, "noDictionaryDimsMapping");
- } catch (Exception e) {
- throw new KettleException(BaseMessages
- .getString(pkg, "CarbonMDKeyStepMeta.Exception.UnexpectedErrorInReadingStepInfo"), e);
- }
- }
-
- @Override
- public StepInterface getStep(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr,
- TransMeta transMeta, Trans trans) {
- return new MDKeyGenStep(stepMeta, stepDataInterface, copyNr, transMeta, trans);
- }
-
- @Override
- public void check(List<CheckResultInterface> remarks, TransMeta transMeta, StepMeta stepMeta,
- RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info) {
- CarbonDataProcessorUtil.checkResult(remarks, stepMeta, input);
- }
-
- @Override public StepDataInterface getStepData() {
- return new MDKeyGenStepData();
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public void setAggregateLevels(String aggregateLevels) {
- this.aggregateLevels = aggregateLevels;
- }
-
- public Map<String, GenericDataType> getComplexTypes() {
- return complexTypes;
- }
-
- public void setNumberOfCores(String numberOfCores) {
- this.numberOfCores = numberOfCores;
- }
-
- /**
- * @return the databaseName
- */
- public String getDatabaseName() {
- return databaseName;
- }
-
- /**
- * @param databaseName the databaseName to set
- */
- public void setDatabaseName(String databaseName) {
- this.databaseName = databaseName;
- }
-
- /**
- * @return the measureCount
- */
- public int getMeasureCount() {
- return Integer.parseInt(measureCount);
- }
-
- /**
- * @param measureCount the measureCount to set
- */
- public void setMeasureCount(String measureCount) {
- this.measureCount = measureCount;
- }
-
- /**
- * @return the dimensionCount
- */
- public int getDimensionCount() {
- return Integer.parseInt(dimensionCount);
- }
-
- /**
- * @param dimensionCount the dimensionCount to set
- */
- public void setDimensionCount(String dimensionCount) {
- this.dimensionCount = dimensionCount;
- }
-
- /**
- * @return the complexDimsCount
- */
- public int getComplexDimsCount() {
- return Integer.parseInt(complexDimsCount);
- }
-
- /**
- * @param complexDimsCount the complexDimsCount to set
- */
- public void setComplexDimsCount(String complexDimsCount) {
- this.complexDimsCount = complexDimsCount;
- }
-
- /**
- * @param complexTypeString the complexTypeString to set
- */
- public void setComplexTypeString(String complexTypeString) {
- this.complexTypeString = complexTypeString;
- }
-
- /**
- * @return
- */
- public String getNoDictionaryDims() {
- return noDictionaryDims;
- }
-
- /**
- * @param noDictionaryDims
- */
- public void setNoDictionaryDims(String noDictionaryDims) {
- this.noDictionaryDims = noDictionaryDims;
- }
-
- /**
- * @return the noDictionaryCount
- */
- public int getNoDictionaryCount() {
- return noDictionaryCount;
- }
-
- /**
- * @param noDictionaryCount the noDictionaryCount to set
- */
- public void setNoDictionaryCount(int noDictionaryCount) {
- this.noDictionaryCount = noDictionaryCount;
- }
-
- public void setColumnGroupsString(String columnGroups) {
- this.columnGroupsString = columnGroups;
-
- }
-
- public void initialize() {
- complexTypes = getComplexTypesMap(complexTypeString);
- }
-
- private Map<String, GenericDataType> getComplexTypesMap(String complexTypeString) {
- if (null == complexTypeString) {
- return new LinkedHashMap<>();
- }
- Map<String, GenericDataType> complexTypesMap = new LinkedHashMap<String, GenericDataType>();
- String[] hierarchies = complexTypeString.split(CarbonCommonConstants.SEMICOLON_SPC_CHARACTER);
- for (int i = 0; i < hierarchies.length; i++) {
- String[] levels = hierarchies[i].split(CarbonCommonConstants.HASH_SPC_CHARACTER);
- String[] levelInfo = levels[0].split(CarbonCommonConstants.COLON_SPC_CHARACTER);
- GenericDataType g = levelInfo[1].equals(CarbonCommonConstants.ARRAY) ?
- new ArrayDataType(levelInfo[0], "", levelInfo[3]) :
- new StructDataType(levelInfo[0], "", levelInfo[3]);
- complexTypesMap.put(levelInfo[0], g);
- for (int j = 1; j < levels.length; j++) {
- levelInfo = levels[j].split(CarbonCommonConstants.COLON_SPC_CHARACTER);
- switch (levelInfo[1]) {
- case CarbonCommonConstants.ARRAY:
- g.addChildren(new ArrayDataType(levelInfo[0], levelInfo[2], levelInfo[3]));
- break;
- case CarbonCommonConstants.STRUCT:
- g.addChildren(new StructDataType(levelInfo[0], levelInfo[2], levelInfo[3]));
- break;
- default:
- g.addChildren(new PrimitiveDataType(levelInfo[0], levelInfo[2], levelInfo[3],
- Integer.parseInt(levelInfo[4])));
- }
- }
- }
- return complexTypesMap;
- }
-
- public void setMeasureDataType(String measureDataType) {
- this.measureDataType = measureDataType;
- }
-
- /**
- * @return
- */
- public int getTaskNo() {
- return Integer.parseInt(taskNo);
- }
-
- /**
- * @param taskNo
- */
- public void setTaskNo(String taskNo) {
- this.taskNo = taskNo;
- }
-
- /**
- * @return
- */
- public String getFactTimeStamp() {
- return factTimeStamp;
- }
-
- /**
- * @param factTimeStamp
- */
- public void setFactTimeStamp(String factTimeStamp) {
- this.factTimeStamp = factTimeStamp;
- }
-
- /**
- * @return partitionId
- */
- public String getPartitionID() {
- return partitionID;
- }
-
- /**
- * @param partitionID
- */
- public void setPartitionID(String partitionID) {
- this.partitionID = partitionID;
- }
-
- /**
- * return segmentId
- *
- * @return
- */
- public String getSegmentId() {
- return segmentId;
- }
-
- /**
- * set segment Id
- *
- * @param segmentId
- */
- public void setSegmentId(String segmentId) {
- this.segmentId = segmentId;
- }
-
- /**
- * @return the noDictionaryDimsMapping
- */
- public String getNoDictionaryDimsMapping() {
- return noDictionaryDimsMapping;
- }
-
- /**
- * @param noDictionaryDimsMapping the noDictionaryDimsMapping to set
- */
- public void setNoDictionaryDimsMapping(String noDictionaryDimsMapping) {
- this.noDictionaryDimsMapping = noDictionaryDimsMapping;
- }
- /**
- * @return isUseInvertedIndex
- */
- public String getIsUseInvertedIndex() {
- return isUseInvertedIndex;
- }
-
- /**
- * @param isUseInvertedIndex the bool array whether use inverted index to set
- */
- public void setIsUseInvertedIndex(String isUseInvertedIndex) {
- this.isUseInvertedIndex = isUseInvertedIndex;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileData.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileData.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileData.java
deleted file mode 100644
index 024a0b9..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileData.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.mdkeygen.file;
-
-import org.apache.carbondata.core.writer.HierarchyValueWriterForCSV;
-
-public class FileData extends FileManager {
-
- /**
- * Store Path
- */
- private String storePath;
-
- /**
- * hierarchyValueWriter
- */
- private HierarchyValueWriterForCSV hierarchyValueWriter;
-
- public FileData(String fileName, String storePath) {
- this.fileName = fileName;
- this.storePath = storePath;
- }
-
- /**
- * @return Returns the carbonDataFileTempPath.
- */
- public String getFileName() {
- return fileName;
- }
-
- /**
- * @return Returns the storePath.
- */
- public String getStorePath() {
- return storePath;
- }
-
- /**
- * get Hierarchy Value writer
- *
- * @return
- */
- public HierarchyValueWriterForCSV getHierarchyValueWriter() {
- return hierarchyValueWriter;
- }
-
- /**
- * Set Hierarchy Value Writer.
- *
- * @param hierarchyValueWriter
- */
- public void setHierarchyValueWriter(HierarchyValueWriterForCSV hierarchyValueWriter) {
- this.hierarchyValueWriter = hierarchyValueWriter;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileManager.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileManager.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileManager.java
deleted file mode 100644
index c867251..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileManager.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.mdkeygen.file;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-
-public class FileManager implements IFileManagerComposite {
- /**
- * listOfFileData, composite parent which holds the different objects
- */
- protected List<IFileManagerComposite> listOfFileData =
- new ArrayList<IFileManagerComposite>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- protected String fileName;
-
- @Override public void add(IFileManagerComposite customData) {
- listOfFileData.add(customData);
- }
-
- @Override public void remove(IFileManagerComposite customData) {
- listOfFileData.remove(customData);
-
- }
-
- @Override public IFileManagerComposite get(int i) {
- return listOfFileData.get(i);
- }
-
- @Override public void setName(String name) {
- this.fileName = name;
- }
-
- /**
- * Return the size
- */
- public int size() {
- return listOfFileData.size();
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/IFileManagerComposite.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/IFileManagerComposite.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/IFileManagerComposite.java
deleted file mode 100644
index 15e06a5..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/IFileManagerComposite.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.mdkeygen.file;
-
-public interface IFileManagerComposite {
- /**
- * Add the data which can be either row Folder(Composite) or File
- *
- * @param customData
- */
- void add(IFileManagerComposite customData);
-
- /**
- * Remove the CustomData type object from the IFileManagerComposite object hierarchy.
- *
- * @param customData
- */
- void remove(IFileManagerComposite customData);
-
- /**
- * get the CustomData type object name
- *
- * @return CustomDataIntf type
- */
- IFileManagerComposite get(int i);
-
- /**
- * set the CustomData type object name
- *
- * @param name
- */
- void setName(String name);
-
- /**
- * Get the size
- *
- * @return
- */
- int size();
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/messages/messages_en_US.properties
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/messages/messages_en_US.properties b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/messages/messages_en_US.properties
deleted file mode 100644
index 3608e57..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/messages/messages_en_US.properties
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-CarbonMDkeyDialog.Shell.Title=Carbon MDkey Generator Step
-CarbonMDkeyDialog.TableName.Label=Table Name
-CarbonMDkeyDialog.SchemaFieldName.Label=Aggregate Levels
-CarbonMDkeyDialog.StoreLocation.Label=Store Location
-CarbonMDkeyDialog.NumberOfCores.Label=Number Of Cores
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/merger/exeception/SliceMergerException.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/exeception/SliceMergerException.java b/processing/src/main/java/org/apache/carbondata/processing/merger/exeception/SliceMergerException.java
deleted file mode 100644
index 3ae3604..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/exeception/SliceMergerException.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.merger.exeception;
-
-import java.util.Locale;
-
-public class SliceMergerException extends Exception {
-
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * The Error message.
- */
- private String msg = "";
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public SliceMergerException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public SliceMergerException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStep.java b/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStep.java
deleted file mode 100644
index 4562793..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStep.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.merger.step;
-
-import java.io.File;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.common.logging.impl.StandardLogService;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
-
-import org.pentaho.di.core.exception.KettleException;
-import org.pentaho.di.core.row.RowMetaInterface;
-import org.pentaho.di.trans.Trans;
-import org.pentaho.di.trans.TransMeta;
-import org.pentaho.di.trans.step.BaseStep;
-import org.pentaho.di.trans.step.StepDataInterface;
-import org.pentaho.di.trans.step.StepMeta;
-import org.pentaho.di.trans.step.StepMetaInterface;
-
-public class CarbonSliceMergerStep extends BaseStep {
-
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CarbonSliceMergerStep.class.getName());
- /**
- * carbon data writer step data class
- */
- private CarbonSliceMergerStepData data;
-
- /**
- * carbon data writer step meta
- */
- private CarbonSliceMergerStepMeta meta;
-
- /**
- * readCounter
- */
- private long readCounter;
-
- /**
- * writeCounter
- */
- private long writeCounter;
-
- /**
- * CarbonSliceMergerStep Constructor
- *
- * @param stepMeta stepMeta
- * @param stepDataInterface stepDataInterface
- * @param copyNr copyNr
- * @param transMeta transMeta
- * @param trans trans
- */
- public CarbonSliceMergerStep(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr,
- TransMeta transMeta, Trans trans) {
- super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
- }
-
- /**
- * Perform the equivalent of processing one row. Typically this means
- * reading a row from input (getRow()) and passing a row to output
- * (putRow)).
- *
- * @param smi The steps metadata to work with
- * @param sdi The steps temporary working data to work with (database
- * connections, result sets, caches, temporary variables, etc.)
- * @return false if no more rows can be processed or an error occurred.
- * @throws KettleException
- */
- public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
- try {
- // carbon data writer step meta
- meta = (CarbonSliceMergerStepMeta) smi;
- StandardLogService.setThreadName(StandardLogService.getPartitionID(meta.getTableName()),
- null);
- // carbon data writer step data
- data = (CarbonSliceMergerStepData) sdi;
-
- // get row from previous step, blocks when needed!
- Object[] row = getRow();
- // if row is null then there is no more incoming data
- if (null == row) {
- renameFolders();
-
- LOGGER.info("Record Procerssed For table: " + meta.getTabelName());
- String logMessage =
- "Summary: Carbon Slice Merger Step: Read: " + readCounter + ": Write: " + writeCounter;
- LOGGER.info(logMessage);
- // step processing is finished
- setOutputDone();
- // return false
- return false;
- }
-
- if (first) {
- first = false;
- if (getInputRowMeta() != null) {
- this.data.setOutputRowMeta((RowMetaInterface) getInputRowMeta().clone());
- this.meta.getFields(data.getOutputRowMeta(), getStepname(), null, null, this);
- }
- }
- readCounter++;
- } catch (Exception ex) {
- LOGGER.error(ex);
- throw new RuntimeException(ex);
- }
- return true;
- }
-
- private void renameFolders() {
- CarbonDataProcessorUtil.renameBadRecordsFromInProgressToNormal(
- meta.getDatabaseName() + File.separator + meta.getTableName() + File.separator + meta
- .getTaskNo());
- }
-
- /**
- * Initialize and do work where other steps need to wait for...
- *
- * @param smi The metadata to work with
- * @param sdi The data to initialize
- * @return step initialize or not
- */
- public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
- meta = (CarbonSliceMergerStepMeta) smi;
- data = (CarbonSliceMergerStepData) sdi;
- return super.init(smi, sdi);
- }
-
- /**
- * Dispose of this step: close files, empty logs, etc.
- *
- * @param smi The metadata to work with
- * @param sdi The data to dispose of
- */
- public void dispose(StepMetaInterface smi, StepDataInterface sdi) {
- meta = (CarbonSliceMergerStepMeta) smi;
- data = (CarbonSliceMergerStepData) sdi;
- super.dispose(smi, sdi);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e6b60907/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepData.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepData.java b/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepData.java
deleted file mode 100644
index b989046..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepData.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.merger.step;
-
-import org.pentaho.di.core.row.RowMetaInterface;
-import org.pentaho.di.trans.step.BaseStepData;
-import org.pentaho.di.trans.step.StepDataInterface;
-
-public class CarbonSliceMergerStepData extends BaseStepData implements StepDataInterface {
- /**
- * outputRowMeta
- */
- private RowMetaInterface outputRowMeta;
-
- public CarbonSliceMergerStepData() {
- super();
- }
-
- public RowMetaInterface getOutputRowMeta() {
- return outputRowMeta;
- }
-
- public void setOutputRowMeta(RowMetaInterface outputRowMeta) {
- this.outputRowMeta = outputRowMeta;
- }
-}