You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2017/01/16 14:53:21 UTC
[44/52] [partial] incubator-carbondata git commit: move core package
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
deleted file mode 100644
index ac1caca..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
+++ /dev/null
@@ -1,405 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.metadata.schema.table.column;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-
-/**
- * Store the information about the column meta data present the table
- */
-public class ColumnSchema implements Serializable {
-
- /**
- * serialization version
- */
- private static final long serialVersionUID = 7676766554874863763L;
-
- /**
- * dataType
- */
- private DataType dataType;
- /**
- * Name of the column. If it is a complex data type, we follow a naming rule
- * grand_parent_column.parent_column.child_column
- * For Array types, two columns will be stored one for
- * the array type and one for the primitive type with
- * the name parent_column.value
- */
- private String columnName;
-
- /**
- * Unique ID for a column. if this is dimension,
- * it is an unique ID that used in dictionary
- */
- private String columnUniqueId;
-
- /**
- * column reference id
- */
- private String columnReferenceId;
-
- /**
- * whether it is stored as columnar format or row format
- */
- private boolean isColumnar = true;
-
- /**
- * List of encoding that are chained to encode the data for this column
- */
- private List<Encoding> encodingList;
-
- /**
- * Whether the column is a dimension or measure
- */
- private boolean isDimensionColumn;
-
- /**
- * Whether the column should use inverted index
- */
- private boolean useInvertedIndex = true;
-
- /**
- * The group ID for column used for row format columns,
- * where in columns in each group are chunked together.
- */
- private int columnGroupId = -1;
-
- /**
- * Used when this column contains decimal data.
- */
- private int scale;
-
- private int precision;
-
- private int schemaOrdinal;
- /**
- * Nested fields. Since thrift does not support nested fields,
- * the nesting is flattened to a single list by a depth-first traversal.
- * The children count is used to construct the nested relationship.
- * This field is not set when the element is a primitive type
- */
- private int numberOfChild;
-
- /**
- * used in case of schema restructuring
- */
- private byte[] defaultValue;
-
- /**
- * Column properties
- */
- private Map<String, String> columnProperties;
-
- /**
- * used to define the column visibility of column default is false
- */
- private boolean invisible = false;
-
- /**
- * @return the columnName
- */
- public String getColumnName() {
- return columnName;
- }
-
- /**
- * @param columnName the columnName to set
- */
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
-
- /**
- * @return the columnUniqueId
- */
- public String getColumnUniqueId() {
- return columnUniqueId;
- }
-
- /**
- * @param columnUniqueId the columnUniqueId to set
- */
- public void setColumnUniqueId(String columnUniqueId) {
- this.columnUniqueId = columnUniqueId;
- }
-
- /**
- * @return the isColumnar
- */
- public boolean isColumnar() {
- return isColumnar;
- }
-
- /**
- * @param isColumnar the isColumnar to set
- */
- public void setColumnar(boolean isColumnar) {
- this.isColumnar = isColumnar;
- }
-
- /**
- * @return the isDimensionColumn
- */
- public boolean isDimensionColumn() {
- return isDimensionColumn;
- }
-
- /**
- * @param isDimensionColumn the isDimensionColumn to set
- */
- public void setDimensionColumn(boolean isDimensionColumn) {
- this.isDimensionColumn = isDimensionColumn;
- }
-
- /**
- * the isUseInvertedIndex
- */
- public boolean isUseInvertedIndex() {
- return useInvertedIndex;
- }
-
- /**
- * @param useInvertedIndex the useInvertedIndex to set
- */
- public void setUseInvertedIndex(boolean useInvertedIndex) {
- this.useInvertedIndex = useInvertedIndex;
- }
-
- /**
- * @return the columnGroup
- */
- public int getColumnGroupId() {
- return columnGroupId;
- }
-
- /**
- */
- public void setColumnGroup(int columnGroupId) {
- this.columnGroupId = columnGroupId;
- }
-
- /**
- * @return the scale
- */
- public int getScale() {
- return scale;
- }
-
- /**
- * @param scale the scale to set
- */
- public void setScale(int scale) {
- this.scale = scale;
- }
-
- /**
- * @return the precision
- */
- public int getPrecision() {
- return precision;
- }
-
- /**
- * @param precision the precision to set
- */
- public void setPrecision(int precision) {
- this.precision = precision;
- }
-
- /**
- * @return the numberOfChild
- */
- public int getNumberOfChild() {
- return numberOfChild;
- }
-
- /**
- * @param numberOfChild the numberOfChild to set
- */
- public void setNumberOfChild(int numberOfChild) {
- this.numberOfChild = numberOfChild;
- }
-
- /**
- * @return the defaultValue
- */
- public byte[] getDefaultValue() {
- return defaultValue;
- }
-
- /**
- * @param defaultValue the defaultValue to set
- */
- public void setDefaultValue(byte[] defaultValue) {
- this.defaultValue = defaultValue;
- }
-
- /**
- * hash code method to check get the hashcode based.
- * for generating the hash code only column name and column unique id will considered
- */
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((columnName == null) ? 0 : columnName.hashCode());
- return result;
- }
-
- /**
- * Overridden equals method for columnSchema
- */
- @Override public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof ColumnSchema)) {
- return false;
- }
- ColumnSchema other = (ColumnSchema) obj;
- if (columnName == null) {
- if (other.columnName != null) {
- return false;
- }
- } else if (!columnName.equals(other.columnName)) {
- return false;
- }
- return true;
- }
-
- /**
- * @return the dataType
- */
- public DataType getDataType() {
- return dataType;
- }
-
- /**
- * @param dataType the dataType to set
- */
- public void setDataType(DataType dataType) {
- this.dataType = dataType;
- }
-
- /**
- * @return the encoderList
- */
- public List<Encoding> getEncodingList() {
- return encodingList;
- }
-
- /**
- */
- public void setEncodingList(List<Encoding> encodingList) {
- this.encodingList = encodingList;
- }
-
- /**
- * @param encoding
- * @return true if contains the passing encoding
- */
- public boolean hasEncoding(Encoding encoding) {
- if (encodingList == null || encodingList.isEmpty()) {
- return false;
- } else {
- return encodingList.contains(encoding);
- }
- }
-
- /**
- * @return if DataType is ARRAY or STRUCT, this method return true, else
- * false.
- */
- public Boolean isComplex() {
- if (DataType.ARRAY.equals(this.getDataType()) || DataType.STRUCT.equals(this.getDataType())) {
- return true;
- } else {
- return false;
- }
- }
-
- /**
- * @param columnProperties
- */
- public void setColumnProperties(Map<String, String> columnProperties) {
- this.columnProperties = columnProperties;
- }
-
- /**
- * @param property
- * @return
- */
- public String getColumnProperty(String property) {
- if (null != columnProperties) {
- return columnProperties.get(property);
- }
- return null;
- }
-
- /**
- * return columnproperties
- */
- public Map<String, String> getColumnProperties() {
- return columnProperties;
- }
- /**
- * return the visibility
- * @return
- */
- public boolean isInvisible() {
- return invisible;
- }
-
- /**
- * set the visibility
- * @param invisible
- */
- public void setInvisible(boolean invisible) {
- this.invisible = invisible;
- }
-
- /**
- * @return columnReferenceId
- */
- public String getColumnReferenceId() {
- return columnReferenceId;
- }
-
- /**
- * @param columnReferenceId
- */
- public void setColumnReferenceId(String columnReferenceId) {
- this.columnReferenceId = columnReferenceId;
- }
-
- public int getSchemaOrdinal() {
- return schemaOrdinal;
- }
-
- public void setSchemaOrdinal(int schemaOrdinal) {
- this.schemaOrdinal = schemaOrdinal;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonSharedDictionaryPath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonSharedDictionaryPath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonSharedDictionaryPath.java
deleted file mode 100644
index 7be92bc..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonSharedDictionaryPath.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.path;
-
-import java.io.File;
-
-/**
- * Helps to get Shared dimension files path.
- */
-public class CarbonSharedDictionaryPath {
-
- private static final String SHAREDDIM_DIR = "SharedDictionary";
- private static final String DICTIONARY_EXT = ".dict";
- private static final String DICTIONARY_META_EXT = ".dictmeta";
- private static final String SORT_INDEX_EXT = ".sortindex";
-
- /***
- * @param storePath store path
- * @param databaseName data base name
- * @param columnId unique column identifier
- * @return absolute path of shared dictionary file
- */
- public static String getDictionaryFilePath(String storePath, String databaseName,
- String columnId) {
- return getSharedDictionaryDir(storePath, databaseName) + File.separator + columnId
- + DICTIONARY_EXT;
- }
-
- /***
- * @param storePath store path
- * @param databaseName data base name
- * @param columnId unique column identifier
- * @return absolute path of shared dictionary meta file
- */
- public static String getDictionaryMetaFilePath(String storePath, String databaseName,
- String columnId) {
- return getSharedDictionaryDir(storePath, databaseName) + File.separator + columnId
- + DICTIONARY_META_EXT;
- }
-
- /***
- * @param storePath store path
- * @param databaseName data base name
- * @param columnId unique column identifier
- * @return absolute path of shared dictionary sort index file
- */
- public static String getSortIndexFilePath(String storePath, String databaseName,
- String columnId) {
- return getSharedDictionaryDir(storePath, databaseName) + File.separator + columnId
- + SORT_INDEX_EXT;
- }
-
- private static String getSharedDictionaryDir(String storePath, String databaseName) {
- return storePath + File.separator + databaseName + File.separator + SHAREDDIM_DIR;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonStorePath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonStorePath.java
deleted file mode 100644
index afe4d9a..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonStorePath.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.path;
-
-import java.io.File;
-
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
-
-import org.apache.hadoop.fs.Path;
-
-/**
- * Helps to get Store content paths.
- */
-public class CarbonStorePath extends Path {
-
- private String storePath;
-
- public CarbonStorePath(String storePathString) {
- super(storePathString);
- this.storePath = storePathString;
- }
-
- /**
- * gets CarbonTablePath object to manage table paths
- */
- public static CarbonTablePath getCarbonTablePath(String storePath,
- CarbonTableIdentifier tableIdentifier) {
- CarbonTablePath carbonTablePath = new CarbonTablePath(tableIdentifier,
- storePath + File.separator + tableIdentifier.getDatabaseName() + File.separator
- + tableIdentifier.getTableName());
- return carbonTablePath;
- }
-
- public static CarbonTablePath getCarbonTablePath(String storePath,
- String dbName, String tableName) {
- return new CarbonTablePath(storePath, dbName, tableName);
- }
-
- public static CarbonTablePath getCarbonTablePath(AbsoluteTableIdentifier identifier) {
- CarbonTableIdentifier id = identifier.getCarbonTableIdentifier();
- return new CarbonTablePath(id, identifier.getTablePath());
- }
-
- /**
- * gets CarbonTablePath object to manage table paths
- */
- public CarbonTablePath getCarbonTablePath(CarbonTableIdentifier tableIdentifier) {
- return CarbonStorePath.getCarbonTablePath(storePath, tableIdentifier);
- }
-
- @Override public boolean equals(Object o) {
- if (!(o instanceof CarbonStorePath)) {
- return false;
- }
- CarbonStorePath path = (CarbonStorePath)o;
- return storePath.equals(path.storePath) && super.equals(o);
- }
-
- @Override public int hashCode() {
- return super.hashCode() + storePath.hashCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
deleted file mode 100644
index a5c9289..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
+++ /dev/null
@@ -1,663 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.path;
-
-import java.io.File;
-
-import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.INVALID_SEGMENT_ID;
-
-import org.apache.hadoop.fs.Path;
-
-/**
- * Helps to get Table content paths.
- */
-public class CarbonTablePath extends Path {
-
- protected static final String METADATA_DIR = "Metadata";
- protected static final String DICTIONARY_EXT = ".dict";
- protected static final String DICTIONARY_META_EXT = ".dictmeta";
- protected static final String SORT_INDEX_EXT = ".sortindex";
- protected static final String SCHEMA_FILE = "schema";
- protected static final String TABLE_STATUS_FILE = "tablestatus";
- protected static final String TABLE_UPDATE_STATUS_FILE = "tableupdatestatus";
- protected static final String FACT_DIR = "Fact";
- protected static final String SEGMENT_PREFIX = "Segment_";
- protected static final String PARTITION_PREFIX = "Part";
- protected static final String CARBON_DATA_EXT = ".carbondata";
- protected static final String CARBON_DELTE_DELTA_EXT = ".deletedelta";
- protected static final String CARBON_UPDATE_DELTA_EXT = ".updatedelta";
- protected static final String DATA_PART_PREFIX = "part-";
- protected static final String INDEX_FILE_EXT = ".carbonindex";
- protected static final String DELETE_DELTA_FILE_EXT = ".deletedelta";
-
- protected String tablePath;
- protected CarbonTableIdentifier carbonTableIdentifier;
-
- /**
- *
- * @param carbonTableIdentifier
- * @param tablePathString
- */
- public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String tablePathString) {
- super(tablePathString);
- this.carbonTableIdentifier = carbonTableIdentifier;
- this.tablePath = tablePathString;
- }
-
- public CarbonTablePath(String storePath, String dbName, String tableName) {
- super(storePath + File.separator + dbName + File.separator + tableName);
- this.carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "");
- this.tablePath = storePath + File.separator + dbName + File.separator + tableName;
- }
-
- /**
- * The method returns the folder path containing the carbon file.
- *
- * @param carbonFilePath
- */
- public static String getFolderContainingFile(String carbonFilePath) {
- return carbonFilePath.substring(0, carbonFilePath.lastIndexOf(File.separator));
- }
-
- /**
- * @param columnId unique column identifier
- * @return name of dictionary file
- */
- public static String getDictionaryFileName(String columnId) {
- return columnId + DICTIONARY_EXT;
- }
-
- /**
- * whether carbonFile is dictionary file or not
- *
- * @param carbonFile
- * @return
- */
- public static Boolean isDictionaryFile(CarbonFile carbonFile) {
- return (!carbonFile.isDirectory()) && (carbonFile.getName().endsWith(DICTIONARY_EXT));
- }
-
- /**
- * check if it is carbon data file matching extension
- *
- * @param fileNameWithPath
- * @return boolean
- */
- public static boolean isCarbonDataFile(String fileNameWithPath) {
- int pos = fileNameWithPath.lastIndexOf('.');
- if (pos != -1) {
- return fileNameWithPath.substring(pos).startsWith(CARBON_DATA_EXT);
- }
- return false;
- }
- /**
- * check if it is carbon data file matching extension
- *
- * @param fileNameWithPath
- * @return boolean
- */
- public static boolean isCarbonDataFileOrUpdateFile(String fileNameWithPath) {
- int pos = fileNameWithPath.lastIndexOf('.');
- if (pos != -1) {
- return fileNameWithPath.substring(pos).startsWith(CARBON_DATA_EXT) || fileNameWithPath
- .substring(pos).startsWith(CARBON_UPDATE_DELTA_EXT);
- }
- return false;
- }
-
- /**
- * check if it is carbon index file matching extension
- *
- * @param fileNameWithPath
- * @return boolean
- */
- public static boolean isCarbonIndexFile(String fileNameWithPath) {
- int pos = fileNameWithPath.lastIndexOf('.');
- if (pos != -1) {
- return fileNameWithPath.substring(pos).startsWith(INDEX_FILE_EXT);
- }
- return false;
- }
-
- /**
- * gets table path
- */
- public String getPath() {
- return tablePath;
- }
-
- /**
- * @param columnId unique column identifier
- * @return absolute path of dictionary file
- */
- public String getDictionaryFilePath(String columnId) {
- return getMetaDataDir() + File.separator + getDictionaryFileName(columnId);
- }
-
- /**
- * @return it return relative directory
- */
- public String getRelativeDictionaryDirectory() {
- return carbonTableIdentifier.getDatabaseName() + File.separator + carbonTableIdentifier
- .getTableName();
- }
-
- /**
- * This method will return the metadata directory location for a table
- *
- * @return
- */
- public String getMetadataDirectoryPath() {
- return getMetaDataDir();
- }
-
- /**
- * @param columnId unique column identifier
- * @return absolute path of dictionary meta file
- */
- public String getDictionaryMetaFilePath(String columnId) {
- return getMetaDataDir() + File.separator + columnId + DICTIONARY_META_EXT;
- }
-
- /**
- * @param columnId unique column identifier
- * @return absolute path of sort index file
- */
- public String getSortIndexFilePath(String columnId) {
- return getMetaDataDir() + File.separator + columnId + SORT_INDEX_EXT;
- }
-
- /**
- *
- * @param columnId
- * @param dictOffset
- * @return absolute path of sortindex with appeneded dictionary offset
- */
- public String getSortIndexFilePath(String columnId, long dictOffset) {
- return getMetaDataDir() + File.separator + columnId + "_" + dictOffset + SORT_INDEX_EXT;
- }
-
- /**
- * @return absolute path of schema file
- */
- public String getSchemaFilePath() {
- return getMetaDataDir() + File.separator + SCHEMA_FILE;
- }
-
- /**
- * return the schema file path
- * @param tablePath path to table files
- * @return schema file path
- */
- public static String getSchemaFilePath(String tablePath) {
- return tablePath + File.separator + METADATA_DIR + File.separator + SCHEMA_FILE;
- }
-
- /**
- * @return absolute path of table status file
- */
- public String getTableStatusFilePath() {
- return getMetaDataDir() + File.separator + TABLE_STATUS_FILE;
- }
-
- /**
- * @return absolute path of table update status file
- */
- public String getTableUpdateStatusFilePath() {
- return getMetaDataDir() + File.separator + TABLE_UPDATE_STATUS_FILE;
- }
-
- /**
- * Gets absolute path of data file
- *
- * @param partitionId unique partition identifier
- * @param segmentId unique partition identifier
- * @param filePartNo data file part number
- * @param factUpdateTimeStamp unique identifier to identify an update
- * @return absolute path of data file stored in carbon data format
- */
- public String getCarbonDataFilePath(String partitionId, String segmentId, Integer filePartNo,
- Integer taskNo, int bucketNumber, String factUpdateTimeStamp) {
- return getSegmentDir(partitionId, segmentId) + File.separator + getCarbonDataFileName(
- filePartNo, taskNo, bucketNumber, factUpdateTimeStamp);
- }
-
- /**
- * Below method will be used to get the index file present in the segment folder
- * based on task id
- *
- * @param taskId task id of the file
- * @param partitionId partition number
- * @param segmentId segment number
- * @return full qualified carbon index path
- */
- public String getCarbonIndexFilePath(final String taskId, final String partitionId,
- final String segmentId, final String bucketNumber) {
- String segmentDir = getSegmentDir(partitionId, segmentId);
- CarbonFile carbonFile =
- FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
-
- CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().startsWith(taskId + "-" + bucketNumber) && file.getName()
- .endsWith(INDEX_FILE_EXT);
- }
- });
- if (files.length > 0) {
- return files[0].getAbsolutePath();
- } else {
- throw new RuntimeException("Missing Carbon index file for partition["
- + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId
- + "]");
- }
- }
- /**
- * Below method will be used to get the index file present in the segment folder
- * based on task id
- *
- * @param taskId task id of the file
- * @param partitionId partition number
- * @param segmentId segment number
- * @return full qualified carbon index path
- */
- public String getCarbonUpdatedIndexFilePath(final String taskId, final String partitionId,
- final String segmentId) {
- String segmentDir = getSegmentDir(partitionId, segmentId);
- CarbonFile carbonFile =
- FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
-
- CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().startsWith(taskId) && file.getName().endsWith(INDEX_FILE_EXT);
- }
- });
- if (files.length > 0) {
- return files[0].getAbsolutePath();
- } else {
- throw new RuntimeException(
- "Missing Carbon Updated index file for partition[" + partitionId
- + "] Segment[" + segmentId + "], taskId[" + taskId + "]");
- }
- }
-
- /**
- * Below method will be used to get the index file present in the segment folder
- * based on task id
- *
- * @param taskId task id of the file
- * @param partitionId partition number
- * @param segmentId segment number
- * @return full qualified carbon index path
- */
- public String getCarbonDeleteDeltaFilePath(final String taskId, final String partitionId,
- final String segmentId) {
- String segmentDir = getSegmentDir(partitionId, segmentId);
- CarbonFile carbonFile =
- FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
-
- CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().startsWith(taskId) && file.getName().endsWith(DELETE_DELTA_FILE_EXT);
- }
- });
- if (files.length > 0) {
- return files[0].getAbsolutePath();
- } else {
- throw new RuntimeException(
- "Missing Carbon delete delta file index file for partition["
- + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId
- + "]");
- }
- }
-
- /**
- * Gets absolute path of data file
- *
- * @param partitionId unique partition identifier
- * @param segmentId unique partition identifier
- * @return absolute path of data file stored in carbon data format
- */
- public String getCarbonDataDirectoryPath(String partitionId, String segmentId) {
- return getSegmentDir(partitionId, segmentId);
- }
-
- /**
- * Gets data file name only with out path
- *
- * @param filePartNo data file part number
- * @param taskNo task identifier
- * @param factUpdateTimeStamp unique identifier to identify an update
- * @return gets data file name only with out path
- */
- public String getCarbonDataFileName(Integer filePartNo, Integer taskNo, int bucketNumber,
- String factUpdateTimeStamp) {
- return DATA_PART_PREFIX + filePartNo + "-" + taskNo + "-" + bucketNumber + "-"
- + factUpdateTimeStamp + CARBON_DATA_EXT;
- }
-
- /**
- * Below method will be used to get the carbon index filename
- *
- * @param taskNo task number
- * @param factUpdatedTimeStamp time stamp
- * @return filename
- */
- public String getCarbonIndexFileName(int taskNo, int bucketNumber, String factUpdatedTimeStamp) {
- return taskNo + "-" + bucketNumber + "-" + factUpdatedTimeStamp + INDEX_FILE_EXT;
- }
-
- /**
- * Below method will be used to get the carbon index filename
- *
- * @param taskNo task number
- * @param factUpdatedTimeStamp time stamp
- * @return filename
- */
- public String getCarbonIndexFileName(int taskNo, String factUpdatedTimeStamp,
- String indexFileExtension) {
- return taskNo + "-" + factUpdatedTimeStamp + indexFileExtension;
- }
-
- private String getSegmentDir(String partitionId, String segmentId) {
- return getPartitionDir(partitionId) + File.separator + SEGMENT_PREFIX + segmentId;
- }
-
- public String getPartitionDir(String partitionId) {
- return getFactDir() + File.separator + PARTITION_PREFIX + partitionId;
- }
-
- private String getMetaDataDir() {
- return tablePath + File.separator + METADATA_DIR;
- }
-
- public String getFactDir() {
- return tablePath + File.separator + FACT_DIR;
- }
-
- @Override public boolean equals(Object o) {
- if (!(o instanceof CarbonTablePath)) {
- return false;
- }
- CarbonTablePath path = (CarbonTablePath) o;
- return tablePath.equals(path.tablePath) && super.equals(o);
- }
-
- @Override public int hashCode() {
- return super.hashCode() + tablePath.hashCode();
- }
-
- /**
- * To manage data file name and composition
- */
- public static class DataFileUtil {
-
- /**
- * gets updated timestamp information from given carbon data file name
- */
- public static String getTimeStampFromFileName(String carbonDataFileName) {
- // Get the timestamp portion of the file.
- String fileName = getFileName(carbonDataFileName);
- int startIndex = fileName.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1;
- int endIndex = fileName.indexOf(".", startIndex);
- return fileName.substring(startIndex, endIndex);
- }
-
-
- /**
- * This will return the timestamp present in the delete delta file.
- * @param fileName
- * @return
- */
- public static String getTimeStampFromDeleteDeltaFile(String fileName) {
- return fileName.substring(fileName.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
- fileName.lastIndexOf("."));
- }
-
- /**
- * This will return the timestamp present in the delete delta file.
- * @param fileName
- * @return
- */
- public static String getBlockNameFromDeleteDeltaFile(String fileName) {
- return fileName.substring(0,
- fileName.lastIndexOf(CarbonCommonConstants.HYPHEN));
- }
-
- /**
- * gets updated timestamp information from given carbon data file name
- */
- public static String getBucketNo(String carbonFilePath) {
- // Get the file name from path
- String fileName = getFileName(carbonFilePath);
- // + 1 for size of "-"
- int firstDashPos = fileName.indexOf("-");
- int secondDash = fileName.indexOf("-", firstDashPos + 1);
- int startIndex = fileName.indexOf("-", secondDash + 1) + 1;
- int endIndex = fileName.indexOf("-", startIndex);
- // to support backward compatibility
- if (startIndex == -1 || endIndex == -1) {
- return "0";
- }
- return fileName.substring(startIndex, endIndex);
- }
-
- /**
- * gets file part number information from given carbon data file name
- */
- public static String getPartNo(String carbonDataFileName) {
- // Get the file name from path
- String fileName = getFileName(carbonDataFileName);
- // + 1 for size of "-"
- int startIndex = fileName.indexOf("-") + 1;
- int endIndex = fileName.indexOf("-", startIndex);
- return fileName.substring(startIndex, endIndex);
- }
-
- /**
- * gets updated timestamp information from given carbon data file name
- */
- public static String getTaskNo(String carbonDataFileName) {
- // Get the file name from path
- String fileName = getFileName(carbonDataFileName);
- // + 1 for size of "-"
- int firstDashPos = fileName.indexOf("-");
- int startIndex = fileName.indexOf("-", firstDashPos + 1) + 1;
- int endIndex = fileName.indexOf("-", startIndex);
- return fileName.substring(startIndex, endIndex);
- }
-
- /**
- * Gets the file name from file path
- */
- private static String getFileName(String carbonDataFileName) {
- int endIndex = carbonDataFileName.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
- if (endIndex > -1) {
- return carbonDataFileName.substring(endIndex + 1, carbonDataFileName.length());
- } else {
- return carbonDataFileName;
- }
- }
-
- /**
- * Gets the file name of the delta files.
- *
- * @param filePartNo
- * @param taskNo
- * @param factUpdateTimeStamp
- * @param Extension
- * @return
- */
- public static String getCarbonDeltaFileName(String filePartNo, String taskNo,
- String factUpdateTimeStamp, String Extension) {
- return DATA_PART_PREFIX + filePartNo + "-" + taskNo + "-" + factUpdateTimeStamp
- + Extension;
- }
- }
-
- /**
- * To manage data path and composition
- */
- public static class DataPathUtil {
-
- /**
- * gets segement id from given absolute data file path
- */
- public static String getSegmentId(String dataFileAbsolutePath) {
- // find segment id from last of data file path
- String tempdataFileAbsolutePath = dataFileAbsolutePath.replace(
- CarbonCommonConstants.WINDOWS_FILE_SEPARATOR, CarbonCommonConstants.FILE_SEPARATOR);
- int endIndex = tempdataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
- // + 1 for size of "/"
- int startIndex = tempdataFileAbsolutePath.lastIndexOf(
- CarbonCommonConstants.FILE_SEPARATOR, endIndex - 1) + 1;
- String segmentDirStr = dataFileAbsolutePath.substring(startIndex, endIndex);
- //identify id in segment_<id>
- String[] segmentDirSplits = segmentDirStr.split("_");
- try {
- if (segmentDirSplits.length == 2) {
- return segmentDirSplits[1];
- }
- } catch (Exception e) {
- return INVALID_SEGMENT_ID;
- }
- return INVALID_SEGMENT_ID;
- }
- }
-
- /**
- * Below method will be used to get sort index file present in mentioned folder
- *
- * @param sortIndexDir directory where sort index file resides
- * @param columnUniqueId columnunique id
- * @return sort index carbon files
- */
- public CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir, final String columnUniqueId) {
- CarbonFile[] files = sortIndexDir.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().startsWith(columnUniqueId) && file.getName().endsWith(SORT_INDEX_EXT);
- }
- });
- return files;
- }
-
- /**
- * returns the carbondata file name
- *
- * @param carbonDataFilePath carbondata file path
- * @return
- */
- public static String getCarbonDataFileName(String carbonDataFilePath) {
- String carbonDataFileName = carbonDataFilePath
- .substring(carbonDataFilePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1,
- carbonDataFilePath.indexOf(CARBON_DATA_EXT));
- return carbonDataFileName;
- }
-
- /**
- *
- * @return carbon data extension
- */
- public static String getCarbonDataExtension() {
- return CARBON_DATA_EXT;
- }
-
- /**
- *
- * @return carbon index extension
- */
- public static String getCarbonIndexExtension() {
- return INDEX_FILE_EXT;
- }
-
- /**
- * This method will remove strings in path and return short block id
- *
- * @param blockId
- * @return shortBlockId
- */
- public static String getShortBlockId(String blockId) {
- return blockId.replace(PARTITION_PREFIX, "")
- .replace(SEGMENT_PREFIX, "")
- .replace(DATA_PART_PREFIX, "")
- .replace(CARBON_DATA_EXT, "");
- }
-
- /**
- * This method will append strings in path and return block id
- *
- * @param shortBlockId
- * @return blockId
- */
- public static String getBlockId(String shortBlockId) {
- String[] splitRecords = shortBlockId.split(CarbonCommonConstants.FILE_SEPARATOR);
- StringBuffer sb = new StringBuffer();
- for (int i = 0; i < splitRecords.length; i++) {
- if (i == 0) {
- sb.append(PARTITION_PREFIX);
- sb.append(splitRecords[i]);
- } else if (i == 1) {
- sb.append(CarbonCommonConstants.FILE_SEPARATOR);
- sb.append(SEGMENT_PREFIX);
- sb.append(splitRecords[i]);
- } else if (i == 2) {
- sb.append(CarbonCommonConstants.FILE_SEPARATOR);
- sb.append(DATA_PART_PREFIX);
- sb.append(splitRecords[i]);
- } else if (i == 3) {
- sb.append(CarbonCommonConstants.FILE_SEPARATOR);
- sb.append(splitRecords[i]);
- sb.append(CARBON_DATA_EXT);
- } else {
- sb.append(CarbonCommonConstants.FILE_SEPARATOR);
- sb.append(splitRecords[i]);
- }
- }
- return sb.toString();
- }
-
-
- /**
- * adds data part prefix to given value
- * @return partition prefix
- */
- public static String addDataPartPrefix(String value) {
- return DATA_PART_PREFIX + value;
- }
-
- /**
- * adds part prefix to given value
- * @return partition prefix
- */
- public static String addPartPrefix(String value) {
- return PARTITION_PREFIX + value;
- }
-
- /**
- * adds part prefix to given value
- * @return partition prefix
- */
- public static String addSegmentPrefix(String value) {
- return SEGMENT_PREFIX + value;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderDummy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderDummy.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderDummy.java
deleted file mode 100644
index 50351a7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderDummy.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.querystatistics;
-
-/**
- * Class will be used to record and log the query statistics
- */
-public class DriverQueryStatisticsRecorderDummy implements QueryStatisticsRecorder{
-
- private DriverQueryStatisticsRecorderDummy() {
-
- }
-
- private static DriverQueryStatisticsRecorderDummy carbonLoadStatisticsImplInstance =
- new DriverQueryStatisticsRecorderDummy();
-
- public static DriverQueryStatisticsRecorderDummy getInstance() {
- return carbonLoadStatisticsImplInstance;
- }
-
- public void recordStatistics(QueryStatistic statistic) {
-
- }
-
- public void logStatistics() {
-
- }
-
- public void logStatisticsAsTableExecutor() {
-
- }
-
- /**
- * Below method will be used to add the statistics
- *
- * @param statistic
- */
- public synchronized void recordStatisticsForDriver(QueryStatistic statistic, String queryId) {
-
- }
-
- /**
- * Below method will be used to show statistic log as table
- */
- public void logStatisticsAsTableDriver() {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderImpl.java
deleted file mode 100644
index d9ad576..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderImpl.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.querystatistics;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-import static org.apache.carbondata.core.util.CarbonUtil.printLine;
-
-import org.apache.commons.lang3.StringUtils;
-
-/**
- * Class will be used to record and log the query statistics
- */
-public class DriverQueryStatisticsRecorderImpl implements QueryStatisticsRecorder{
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DriverQueryStatisticsRecorderImpl.class.getName());
-
- /**
- * singleton QueryStatisticsRecorder for driver
- */
- private Map<String, List<QueryStatistic>> queryStatisticsMap;
-
- /**
- * lock for log statistics table
- */
- private static final Object lock = new Object();
-
- private DriverQueryStatisticsRecorderImpl() {
- // use ConcurrentHashMap, it is thread-safe
- queryStatisticsMap = new ConcurrentHashMap<String, List<QueryStatistic>>();
- }
-
- private static DriverQueryStatisticsRecorderImpl carbonLoadStatisticsImplInstance =
- new DriverQueryStatisticsRecorderImpl();
-
- public static DriverQueryStatisticsRecorderImpl getInstance() {
- return carbonLoadStatisticsImplInstance;
- }
-
- public void recordStatistics(QueryStatistic statistic) {
-
- }
-
- public void logStatistics() {
-
- }
-
- public void logStatisticsAsTableExecutor() {
-
- }
-
- /**
- * Below method will be used to add the statistics
- *
- * @param statistic
- */
- public void recordStatisticsForDriver(QueryStatistic statistic, String queryId) {
- synchronized (lock) {
- // refresh query Statistics Map
- if (queryStatisticsMap.get(queryId) != null) {
- queryStatisticsMap.get(queryId).add(statistic);
- } else {
- List<QueryStatistic> newQueryStatistics = new ArrayList<QueryStatistic>();
- newQueryStatistics.add(statistic);
- queryStatisticsMap.put(queryId, newQueryStatistics);
- }
- }
- }
-
- /**
- * Below method will be used to show statistic log as table
- */
- public void logStatisticsAsTableDriver() {
- synchronized (lock) {
- Iterator<Map.Entry<String, List<QueryStatistic>>> entries =
- queryStatisticsMap.entrySet().iterator();
- while (entries.hasNext()) {
- Map.Entry<String, List<QueryStatistic>> entry = entries.next();
- String queryId = entry.getKey();
- // clear the unknown query statistics
- if(StringUtils.isEmpty(queryId)) {
- entries.remove();
- } else {
- // clear the timeout query statistics
- long interval = System.nanoTime() - Long.parseLong(queryId);
- if (interval > QueryStatisticsConstants.CLEAR_STATISTICS_TIMEOUT) {
- entries.remove();
- } else {
- // print sql_parse_t,load_meta_t,block_allocation_t,block_identification_t
- // or just print block_allocation_t,block_identification_t
- if (entry.getValue().size() >= 2) {
- String tableInfo = collectDriverStatistics(entry.getValue(), queryId);
- if (null != tableInfo) {
- LOGGER.statistic(tableInfo);
- // clear the statistics that has been printed
- entries.remove();
- }
- }
- }
- }
- }
- }
- }
-
- /**
- * Below method will parse queryStatisticsMap and put time into table
- */
- public String collectDriverStatistics(List<QueryStatistic> statisticsList, String queryId) {
- String sql_parse_time = "";
- String load_meta_time = "";
- String load_blocks_time = "";
- String block_allocation_time = "";
- String block_identification_time = "";
- long driver_part_time_tmp = 0L;
- long driver_part_time_tmp2 = 0L;
- long load_blocks_time_tmp = 0L;
- String splitChar = " ";
- try {
- // get statistic time from the QueryStatistic
- for (QueryStatistic statistic : statisticsList) {
- switch (statistic.getMessage()) {
- case QueryStatisticsConstants.SQL_PARSE:
- sql_parse_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.LOAD_META:
- load_meta_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.LOAD_BLOCKS_DRIVER:
- // multi segments will generate multi load_blocks_time
- load_blocks_time_tmp += statistic.getTimeTaken();
- driver_part_time_tmp += statistic.getTimeTaken();
- driver_part_time_tmp2 += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.BLOCK_ALLOCATION:
- block_allocation_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- driver_part_time_tmp2 += statistic.getTimeTaken();
- break;
- case QueryStatisticsConstants.BLOCK_IDENTIFICATION:
- block_identification_time += statistic.getTimeTaken() + splitChar;
- driver_part_time_tmp += statistic.getTimeTaken();
- driver_part_time_tmp2 += statistic.getTimeTaken();
- break;
- default:
- break;
- }
- }
- load_blocks_time = load_blocks_time_tmp + splitChar;
- String driver_part_time = driver_part_time_tmp + splitChar;
- // structure the query statistics info table
- StringBuilder tableInfo = new StringBuilder();
- int len1 = 8;
- int len2 = 20;
- int len3 = 21;
- int len4 = 24;
- String line = "+" + printLine("-", len1) + "+" + printLine("-", len2) + "+" +
- printLine("-", len3) + "+" + printLine("-", len4) + "+";
- String line2 = "|" + printLine(" ", len1) + "+" + printLine("-", len2) + "+" +
- printLine(" ", len3) + "+" + printLine("-", len4) + "+";
- // table header
- tableInfo.append(line).append("\n");
- tableInfo.append("|" + printLine(" ", (len1 - "Module".length())) + "Module" + "|" +
- printLine(" ", (len2 - "Operation Step".length())) + "Operation Step" + "|" +
- printLine(" ", (len3 - "Total Query Cost".length())) + "Total Query Cost" + "|" +
- printLine(" ", (len4 - "Query Cost".length())) + "Query Cost" + "|" + "\n");
- tableInfo.append(line).append("\n");
- // print sql_parse_t,load_meta_t,block_allocation_t,block_identification_t
- if (!StringUtils.isEmpty(sql_parse_time) &&
- !StringUtils.isEmpty(load_meta_time) &&
- !StringUtils.isEmpty(block_allocation_time) &&
- !StringUtils.isEmpty(block_identification_time)) {
- tableInfo.append("|" + printLine(" ", len1) + "|" +
- printLine(" ", (len2 - "SQL parse".length())) + "SQL parse" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - sql_parse_time.length())) + sql_parse_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" + printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" +
- printLine(" ", (len2 - "Load meta data".length())) + "Load meta data" + "|" +
- printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" +
- printLine(" ", (len4 - load_meta_time.length())) +
- load_meta_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" + printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
- printLine(" ", (len2 - "Load blocks driver".length())) +
- "Load blocks driver" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - load_blocks_time.length())) +
- load_blocks_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" + printLine(" ", len1 ) + "|" +
- printLine(" ", (len2 - "Block allocation".length())) + "Block allocation" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - block_allocation_time.length())) +
- block_allocation_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +
- printLine(" ", len1) + "|" +
- printLine(" ", (len2 - "Block identification".length())) +
- "Block identification" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - block_identification_time.length())) +
- block_identification_time + "|" + "\n");
- tableInfo.append(line).append("\n");
-
- // show query statistic as "query id" + "table"
- return "Print query statistic for query id: " + queryId + "\n" + tableInfo.toString();
- } else if (!StringUtils.isEmpty(block_allocation_time) &&
- !StringUtils.isEmpty(block_identification_time)) {
- // when we can't get sql parse time, we only print the last two
- driver_part_time = driver_part_time_tmp2 + splitChar;
- tableInfo.append("|" + printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" +
- printLine(" ", (len2 - "Load blocks driver".length())) +
- "Load blocks driver" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - load_blocks_time.length())) +
- load_blocks_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" + printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
- printLine(" ", (len2 - "Block allocation".length())) + "Block allocation" + "|" +
- printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" +
- printLine(" ", (len4 - block_allocation_time.length())) +
- block_allocation_time + "|" + "\n");
- tableInfo.append(line2).append("\n");
- tableInfo.append("|" +
- printLine(" ", len1) + "|" +
- printLine(" ", (len2 - "Block identification".length())) +
- "Block identification" + "|" +
- printLine(" ", len3) + "|" +
- printLine(" ", (len4 - block_identification_time.length())) +
- block_identification_time + "|" + "\n");
- tableInfo.append(line).append("\n");
-
- // show query statistic as "query id" + "table"
- return "Print query statistic for query id: " + queryId + "\n" + tableInfo.toString();
- }
-
- return null;
- } catch (Exception ex) {
- return "Put statistics into table failed, catch exception: " + ex.getMessage();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
deleted file mode 100644
index 74fd651..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.querystatistics;
-
-import java.io.Serializable;
-
-import org.apache.commons.lang3.StringUtils;
-
-/**
- * Wrapper class to maintain the query statistics for each phase of the query
- */
-public class QueryStatistic implements Serializable {
-
- /**
- * serialization id
- */
- private static final long serialVersionUID = -5667106646135905848L;
-
- /**
- * statistic message
- */
- private String message;
-
- /**
- * total time take of the phase
- */
- private long timeTaken;
-
- /**
- * starttime of the phase
- */
- private long startTime;
-
- /**
- * number of count
- */
- private long count;
-
- public QueryStatistic() {
- this.startTime = System.currentTimeMillis();
- }
-
- /**
- * below method will be used to add the statistic
- *
- * @param message Statistic message
- * @param currentTime current time
- */
- public void addStatistics(String message, long currentTime) {
- this.timeTaken = currentTime - startTime;
- this.message = message;
- }
-
- /**
- * Below method will be used to add fixed time statistic.
- * For example total time taken for scan or result preparation
- *
- * @param message statistic message
- * @param timetaken
- */
- public void addFixedTimeStatistic(String message, long timetaken) {
- this.timeTaken = timetaken;
- this.message = message;
- }
-
- public void addCountStatistic(String message, long count) {
- this.timeTaken = -1;
- this.count = count;
- this.message = message;
- }
-
- /**
- * Below method will be used to get the statistic message, which will
- * be used to log
- *
- * @param queryWithTaskId query with task id to append in the message
- * @return statistic message
- */
- public String getStatistics(String queryWithTaskId) {
- if (StringUtils.isEmpty(queryWithTaskId)) {
- return message + timeTaken;
- }
- return message + " for the taskid : " + queryWithTaskId + " Is : " + timeTaken;
- }
-
- public String getMessage() {
- return this.message;
- }
-
- public long getTimeTaken() {
- return this.timeTaken;
- }
-
- public long getCount() {
- return this.count;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
deleted file mode 100644
index a76dd9b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.carbon.querystatistics;
-
-public interface QueryStatisticsConstants {
-
- // driver side
- String SQL_PARSE = "Time taken to parse sql In Driver Side";
-
- String LOAD_META = "Time taken to load meta data In Driver Side";
-
- String LOAD_BLOCKS_DRIVER = "Time taken to load the Block(s) In Driver Side";
-
- String BLOCK_ALLOCATION = "Total Time taken in block(s) allocation";
-
- String BLOCK_IDENTIFICATION = "Time taken to identify Block(s) to scan";
-
- // executor side
- String EXECUTOR_PART =
- "Total Time taken to execute the query in executor Side";
-
- String LOAD_BLOCKS_EXECUTOR = "Time taken to load the Block(s) In Executor";
-
- String SCAN_BLOCKS_NUM = "The num of blocks scanned";
-
- String SCAN_BLOCKS_TIME = "Time taken to scan blocks";
-
- String LOAD_DICTIONARY = "Time taken to load the Dictionary In Executor";
-
- String PREPARE_RESULT = "Total Time taken to prepare query result";
-
- String RESULT_SIZE = "The size of query result";
-
- String TOTAL_BLOCKLET_NUM = "The num of total blocklet";
-
- String VALID_SCAN_BLOCKLET_NUM = "The num of valid scanned blocklet";
-
- // clear no-use statistics timeout
- long CLEAR_STATISTICS_TIMEOUT = 60 * 1000 * 1000000L;
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsModel.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsModel.java
deleted file mode 100644
index e62d726..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsModel.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.carbon.querystatistics;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class QueryStatisticsModel {
- private QueryStatisticsRecorder recorder;
- private Map<String, QueryStatistic> statisticsTypeAndObjMap =
- new HashMap<String, QueryStatistic>();
-
- public QueryStatisticsRecorder getRecorder() {
- return recorder;
- }
-
- public void setRecorder(QueryStatisticsRecorder recorder) {
- this.recorder = recorder;
- }
-
- public Map<String, QueryStatistic> getStatisticsTypeAndObjMap() {
- return statisticsTypeAndObjMap;
- }
-
- public void setStatisticsTypeAndObjMap(Map<String, QueryStatistic> statisticsTypeAndObjMap) {
- this.statisticsTypeAndObjMap = statisticsTypeAndObjMap;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java
deleted file mode 100644
index 439a9b3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.querystatistics;
-
-/**
- * interface will be used to record and log the query statistics
- */
-public interface QueryStatisticsRecorder {
-
- void recordStatistics(QueryStatistic statistic);
-
- void logStatistics();
-
- void logStatisticsAsTableExecutor();
-
- void recordStatisticsForDriver(QueryStatistic statistic, String queryId);
-
- void logStatisticsAsTableDriver();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java
deleted file mode 100644
index 4d64242..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.querystatistics;
-
-import java.io.Serializable;
-
-/**
- * Class will be used to record and log the query statistics
- */
-public class QueryStatisticsRecorderDummy implements QueryStatisticsRecorder,Serializable {
-
- /**
- * serialization version
- */
- private static final long serialVersionUID = -5719752001674467864L;
-
- public QueryStatisticsRecorderDummy() {
-
- }
-
- /**
- * Below method will be used to add the statistics
- *
- * @param statistic
- */
- public synchronized void recordStatistics(QueryStatistic statistic) {
-
- }
-
- /**
- * Below method will be used to log the statistic
- */
- public void logStatistics() {
-
- }
-
- /**
- * Below method will be used to show statistic log as table
- */
- public void logStatisticsAsTableExecutor() {
-
- }
-
- public void recordStatisticsForDriver(QueryStatistic statistic, String queryId) {
-
- }
-
- public void logStatisticsAsTableDriver() {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderImpl.java
deleted file mode 100644
index 996e7f0..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderImpl.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.querystatistics;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-import static org.apache.carbondata.core.util.CarbonUtil.printLine;
-
-/**
- * Class will be used to record and log the query statistics
- */
-public class QueryStatisticsRecorderImpl implements QueryStatisticsRecorder,Serializable {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(QueryStatisticsRecorderImpl.class.getName());
-
- /**
- * serialization version
- */
- private static final long serialVersionUID = -5719752001674467864L;
-
- /**
- * list for statistics to record time taken
- * by each phase of the query for example aggregation
- * scanning,block loading time etc.
- */
- private List<QueryStatistic> queryStatistics;
-
- /**
- * query with taskd
- */
- private String queryIWthTask;
-
- public QueryStatisticsRecorderImpl(String queryId) {
- queryStatistics = new ArrayList<QueryStatistic>();
- this.queryIWthTask = queryId;
- }
-
- /**
- * Below method will be used to add the statistics
- *
- * @param statistic
- */
- public synchronized void recordStatistics(QueryStatistic statistic) {
- queryStatistics.add(statistic);
- }
-
- /**
- * Below method will be used to log the statistic
- */
- public void logStatistics() {
- for (QueryStatistic statistic : queryStatistics) {
- LOGGER.statistic(statistic.getStatistics(queryIWthTask));
- }
- }
-
- /**
- * Below method will be used to show statistic log as table
- */
- public void logStatisticsAsTableExecutor() {
- String tableInfo = collectExecutorStatistics();
- if (null != tableInfo) {
- LOGGER.statistic(tableInfo);
- }
- }
-
- /**
- * Below method will parse queryStatisticsMap and put time into table
- */
- public String collectExecutorStatistics() {
- String load_blocks_time = "";
- String scan_blocks_time = "";
- String scan_blocks_num = "";
- String load_dictionary_time = "";
- String result_size = "";
- String total_executor_time = "";
- String splitChar = " ";
- String total_blocklet = "";
- String valid_scan_blocklet = "";
- try {
- for (QueryStatistic statistic : queryStatistics) {
- switch (statistic.getMessage()) {
- case QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR:
- load_blocks_time += statistic.getTimeTaken() + splitChar;
- break;
- case QueryStatisticsConstants.SCAN_BLOCKS_TIME:
- scan_blocks_time += statistic.getTimeTaken() + splitChar;
- break;
- case QueryStatisticsConstants.SCAN_BLOCKS_NUM:
- scan_blocks_num += statistic.getCount() + splitChar;
- break;
- case QueryStatisticsConstants.LOAD_DICTIONARY:
- load_dictionary_time += statistic.getTimeTaken() + splitChar;
- break;
- case QueryStatisticsConstants.RESULT_SIZE:
- result_size += statistic.getCount() + splitChar;
- break;
- case QueryStatisticsConstants.EXECUTOR_PART:
- total_executor_time += statistic.getTimeTaken() + splitChar;
- break;
- case QueryStatisticsConstants.TOTAL_BLOCKLET_NUM:
- total_blocklet = statistic.getCount() + splitChar;
- break;
- case QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM:
- valid_scan_blocklet = statistic.getCount() + splitChar;
- break;
- default:
- break;
- }
- }
- String headers = "task_id,load_blocks_time,load_dictionary_time,scan_blocks_time," +
- "total_executor_time,scan_blocks_num,total_blocklet," +
- "valid_scan_blocklet,result_size";
- List<String> values = new ArrayList<String>();
- values.add(queryIWthTask);
- values.add(load_blocks_time);
- values.add(load_dictionary_time);
- values.add(scan_blocks_time);
- values.add(total_executor_time);
- values.add(scan_blocks_num);
- values.add(total_blocklet);
- values.add(valid_scan_blocklet);
- values.add(result_size);
- StringBuilder tableInfo = new StringBuilder();
- String[] columns = headers.split(",");
- String line = "";
- String hearLine = "";
- String valueLine = "";
- for (int i = 0; i < columns.length; i++) {
- int len = Math.max(columns[i].length(), values.get(i).length());
- line += "+" + printLine("-", len);
- hearLine += "|" + printLine(" ", len - columns[i].length()) + columns[i];
- valueLine += "|" + printLine(" ", len - values.get(i).length()) + values.get(i);
- }
- // struct table info
- tableInfo.append(line + "+").append("\n");
- tableInfo.append(hearLine + "|").append("\n");
- tableInfo.append(line + "+").append("\n");
- tableInfo.append(valueLine + "|").append("\n");
- tableInfo.append(line + "+").append("\n");
- return "Print query statistic for each task id:" + "\n" + tableInfo.toString();
- } catch (Exception ex) {
- return "Put statistics into table failed, catch exception: " + ex.getMessage();
- }
- }
-
- public void recordStatisticsForDriver(QueryStatistic statistic, String queryId) {
-
- }
-
- public void logStatisticsAsTableDriver() {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
index 96ca2e1..fe8192e 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.compression;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.BigDecimalCompressionFinder;
import org.apache.carbondata.core.util.CompressionFinder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
index c27887d..6a5250a 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.compression;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
index 383a515..345865c 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.compression;
import java.math.BigDecimal;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
index 31f86c0..a74b886 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.compression;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.CompressionFinder;
import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
deleted file mode 100644
index 1f99158..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store;
-
-import java.io.IOException;
-
-public interface FileHolder {
- /**
- * This method will be used to read the byte array from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, long offset, int length) throws IOException;
-
- /**
- * This method will be used to read the byte array from file based on length(number of bytes)
- *
- * @param filePath fully qualified file path
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, int length) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- int readInt(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readLong(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @return read int
- */
- int readInt(String filePath) throws IOException;
-
- /**
- * This method will be used to read long value from file from postion(offset), here
- * length will be always 8 because long byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readDouble(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to close all the streams currently present in the cache
- */
- void finish() throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
deleted file mode 100644
index 80a4374..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store;
-
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-
-/**
- * MeasureDataWrapper, interface.
- */
-public interface MeasureDataWrapper {
- CarbonReadDataHolder[] getValues();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
deleted file mode 100644
index e40dadd..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store;
-
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
-
-public interface NodeMeasureDataStore {
- /**
- * This method will be used to get the writable key array.
- * writable measure data array will hold below information:
- * <size of measure data array><measure data array>
- * total length will be 4 bytes for size + measure data array length
- *
- * @return writable array (compressed or normal)
- */
- byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
-}
\ No newline at end of file