You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/01/04 13:34:55 UTC
[1/9] incubator-carbondata git commit: clean up core
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 82072ee19 -> 2ad623dcb
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
index 214e574..63d68f6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
@@ -177,9 +177,9 @@ public class UnsafeSortDataRows {
* record holder heap and then it will read first record from each file and
* initialize the heap
*
- * @throws CarbonSortKeyAndGroupByException
+ * @throws InterruptedException
*/
- public void startSorting() throws CarbonSortKeyAndGroupByException {
+ public void startSorting() throws InterruptedException {
LOGGER.info("Unsafe based sorting will be used");
if (this.rowPage.getUsedSize() > 0) {
TimSort<UnsafeCarbonRow, IntPointerBuffer> timSort = new TimSort<>(
@@ -223,25 +223,19 @@ public class UnsafeSortDataRows {
/**
* This method will be used to delete sort temp location is it is exites
- *
- * @throws CarbonSortKeyAndGroupByException
*/
- public void deleteSortLocationIfExists() throws CarbonSortKeyAndGroupByException {
+ public void deleteSortLocationIfExists() {
CarbonDataProcessorUtil.deleteSortLocationIfExists(parameters.getTempFileLocation());
}
/**
* Below method will be used to start file based merge
*
- * @throws CarbonSortKeyAndGroupByException
+ * @throws InterruptedException
*/
- private void startFileBasedMerge() throws CarbonSortKeyAndGroupByException {
- try {
- dataSorterAndWriterExecutorService.shutdown();
- dataSorterAndWriterExecutorService.awaitTermination(2, TimeUnit.DAYS);
- } catch (InterruptedException e) {
- throw new CarbonSortKeyAndGroupByException("Problem while shutdown the server ", e);
- }
+ private void startFileBasedMerge() throws InterruptedException {
+ dataSorterAndWriterExecutorService.shutdown();
+ dataSorterAndWriterExecutorService.awaitTermination(2, TimeUnit.DAYS);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
index 735243e..a893ba8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
@@ -36,7 +36,6 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.newflow.sort.unsafe.holder.SortTempChunkHolder;
@@ -357,7 +356,7 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
try {
CarbonUtil.deleteFiles(intermediateFiles);
rowData.clear();
- } catch (CarbonUtilException e) {
+ } catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while deleting the intermediate files");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index 1c59d29..41f7cd2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -20,6 +20,7 @@
package org.apache.carbondata.processing.newflow.steps;
import java.io.File;
+import java.io.IOException;
import java.util.Iterator;
import org.apache.carbondata.common.CarbonIterator;
@@ -33,7 +34,6 @@ import org.apache.carbondata.processing.newflow.DataField;
import org.apache.carbondata.processing.newflow.constants.DataLoadProcessorConstants;
import org.apache.carbondata.processing.newflow.converter.RowConverter;
import org.apache.carbondata.processing.newflow.converter.impl.RowConverterImpl;
-import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.newflow.row.CarbonRow;
import org.apache.carbondata.processing.newflow.row.CarbonRowBatch;
import org.apache.carbondata.processing.surrogatekeysgenerator.csvbased.BadRecordsLogger;
@@ -57,7 +57,7 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
}
@Override
- public void initialize() throws CarbonDataLoadingException {
+ public void initialize() throws IOException {
child.initialize();
BadRecordsLogger badRecordLogger = createBadRecordLogger();
converter = new RowConverterImpl(child.getOutput(), configuration, badRecordLogger);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index 16b203a..23d7af1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -20,6 +20,7 @@
package org.apache.carbondata.processing.newflow.steps;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -39,7 +40,6 @@ import org.apache.carbondata.processing.newflow.DataField;
import org.apache.carbondata.processing.newflow.constants.DataLoadProcessorConstants;
import org.apache.carbondata.processing.newflow.converter.RowConverter;
import org.apache.carbondata.processing.newflow.converter.impl.RowConverterImpl;
-import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.newflow.row.CarbonRow;
import org.apache.carbondata.processing.newflow.row.CarbonRowBatch;
import org.apache.carbondata.processing.surrogatekeysgenerator.csvbased.BadRecordsLogger;
@@ -65,7 +65,7 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
}
@Override
- public void initialize() throws CarbonDataLoadingException {
+ public void initialize() throws IOException {
child.initialize();
BadRecordsLogger badRecordLogger = createBadRecordLogger();
converter = new RowConverterImpl(child.getOutput(), configuration, badRecordLogger);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
index 8318530..6f1c035 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
@@ -19,6 +19,7 @@
package org.apache.carbondata.processing.newflow.steps;
import java.io.File;
+import java.io.IOException;
import java.util.Iterator;
import org.apache.carbondata.common.logging.LogService;
@@ -78,7 +79,7 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
return child.getOutput();
}
- @Override public void initialize() throws CarbonDataLoadingException {
+ @Override public void initialize() throws IOException {
child.initialize();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
index ef43751..b8f08e9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.processing.newflow.steps;
+import java.io.IOException;
import java.util.Iterator;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -53,7 +54,7 @@ public class SortProcessorStepImpl extends AbstractDataLoadProcessorStep {
}
@Override
- public void initialize() throws CarbonDataLoadingException {
+ public void initialize() throws IOException {
child.initialize();
SortParameters sortParameters = SortParameters.createSortParameters(configuration);
boolean offheapsort = Boolean.parseBoolean(CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
index f1b062e..db9d379 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
@@ -33,7 +33,6 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.util.RemoveDictionaryUtil;
@@ -453,7 +452,7 @@ public class IntermediateFileMerger implements Callable<Void> {
}
try {
CarbonUtil.deleteFiles(intermediateFiles);
- } catch (CarbonUtilException e) {
+ } catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while deleting the intermediate files");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
index 9b3bec7..c92b11b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
@@ -20,6 +20,7 @@
package org.apache.carbondata.processing.surrogatekeysgenerator.csvbased;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -27,8 +28,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
@@ -39,12 +38,10 @@ import org.apache.carbondata.core.carbon.ColumnIdentifier;
import org.apache.carbondata.core.carbon.metadata.CarbonMetadata;
import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.core.writer.ByteArrayHolder;
import org.apache.carbondata.core.writer.HierarchyValueWriterForCSV;
import org.apache.carbondata.processing.datatypes.GenericDataType;
@@ -61,12 +58,6 @@ import org.pentaho.di.core.exception.KettleException;
public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKeyGen {
/**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(FileStoreSurrogateKeyGenForCSV.class.getName());
-
- /**
* hierValueWriter
*/
private Map<String, HierarchyValueWriterForCSV> hierValueWriter;
@@ -77,21 +68,11 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
private Map<String, KeyGenerator> keyGenerator;
/**
- * baseStorePath
- */
- private String baseStorePath;
-
- /**
* LOAD_FOLDER
*/
private String loadFolderName;
/**
- * folderList
- */
- private List<CarbonFile> folderList = new ArrayList<CarbonFile>(5);
-
- /**
* primaryKeyStringArray
*/
private String[] primaryKeyStringArray;
@@ -110,10 +91,10 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
/**
* @param columnsInfo
- * @throws KettleException
+ * @throws IOException
*/
public FileStoreSurrogateKeyGenForCSV(ColumnsInfo columnsInfo, String partitionID, int segmentId,
- String taskNo) throws KettleException {
+ String taskNo) throws IOException {
super(columnsInfo);
populatePrimaryKeyarray(dimInsertFileNames, columnsInfo.getPrimaryKeyMap());
this.partitionID = partitionID;
@@ -121,7 +102,6 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
this.taskNo = taskNo;
keyGenerator = new HashMap<String, KeyGenerator>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- baseStorePath = columnsInfo.getBaseStoreLocation();
setStoreFolderWithLoadNumber(
checkAndCreateLoadFolderNumber(columnsInfo.getDatabaseName(),
columnsInfo.getTableName()));
@@ -191,13 +171,13 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
}
private String checkAndCreateLoadFolderNumber(String databaseName,
- String tableName) throws KettleException {
+ String tableName) throws IOException {
String carbonDataDirectoryPath = CarbonDataProcessorUtil
.getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId + "",
false);
boolean isDirCreated = new File(carbonDataDirectoryPath).mkdirs();
if (!isDirCreated) {
- throw new KettleException("Unable to create data load directory" + carbonDataDirectoryPath);
+ throw new IOException("Unable to create data load directory" + carbonDataDirectoryPath);
}
return carbonDataDirectoryPath;
}
@@ -214,7 +194,7 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
/**
* This method will generate cache for all the global dictionaries during data loading.
*/
- private void populateCache() throws KettleException {
+ private void populateCache() throws IOException {
String carbonStorePath =
CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS);
String[] dimColumnNames = columnsInfo.getDimColNames();
@@ -268,7 +248,7 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
}
}
initDictionaryCacheInfo(dictionaryKeys, dictionaryColumnUniqueIdentifiers,
- reverseDictionaryCache, carbonStorePath);
+ reverseDictionaryCache);
}
/**
@@ -277,25 +257,20 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
* @param dictionaryKeys
* @param dictionaryColumnUniqueIdentifiers
* @param reverseDictionaryCache
- * @param carbonStorePath
* @throws KettleException
*/
private void initDictionaryCacheInfo(List<String> dictionaryKeys,
List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers,
- Cache reverseDictionaryCache, String carbonStorePath) throws KettleException {
+ Cache reverseDictionaryCache) throws IOException {
long lruCacheStartTime = System.currentTimeMillis();
- try {
- List reverseDictionaries = reverseDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
- for (int i = 0; i < reverseDictionaries.size(); i++) {
- Dictionary reverseDictionary = (Dictionary) reverseDictionaries.get(i);
- getDictionaryCaches().put(dictionaryKeys.get(i), reverseDictionary);
- updateMaxKeyInfo(dictionaryKeys.get(i), reverseDictionary.getDictionaryChunks().getSize());
- }
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordLruCacheLoadTime(
- (System.currentTimeMillis() - lruCacheStartTime)/1000.0);
- } catch (CarbonUtilException e) {
- throw new KettleException(e.getMessage());
+ List reverseDictionaries = reverseDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
+ for (int i = 0; i < reverseDictionaries.size(); i++) {
+ Dictionary reverseDictionary = (Dictionary) reverseDictionaries.get(i);
+ getDictionaryCaches().put(dictionaryKeys.get(i), reverseDictionary);
+ updateMaxKeyInfo(dictionaryKeys.get(i), reverseDictionary.getDictionaryChunks().getSize());
}
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordLruCacheLoadTime(
+ (System.currentTimeMillis() - lruCacheStartTime)/1000.0);
}
@Override protected int getSurrogateFromStore(String value, int index, Object[] properties)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 3cb984d..cc142e7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -52,10 +52,8 @@ import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.datatypes.ArrayDataType;
import org.apache.carbondata.processing.datatypes.GenericDataType;
@@ -63,7 +61,6 @@ import org.apache.carbondata.processing.datatypes.PrimitiveDataType;
import org.apache.carbondata.processing.datatypes.StructDataType;
import org.apache.carbondata.processing.etl.DataLoadingException;
import org.apache.carbondata.processing.newflow.DataField;
-import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
import org.apache.commons.lang3.ArrayUtils;
import org.pentaho.di.core.CheckResult;
@@ -225,11 +222,8 @@ public final class CarbonDataProcessorUtil {
/**
* This method will be used to delete sort temp location is it is exites
- *
- * @throws CarbonSortKeyAndGroupByException
*/
- public static void deleteSortLocationIfExists(String tempFileLocation)
- throws CarbonSortKeyAndGroupByException {
+ public static void deleteSortLocationIfExists(String tempFileLocation) {
// create new temp file location where this class
//will write all the temp files
File file = new File(tempFileLocation);
@@ -237,43 +231,13 @@ public final class CarbonDataProcessorUtil {
if (file.exists()) {
try {
CarbonUtil.deleteFoldersAndFiles(file);
- } catch (CarbonUtilException e) {
+ } catch (IOException | InterruptedException e ) {
LOGGER.error(e);
}
}
}
/**
- * return the modification TimeStamp Separated by HASH_SPC_CHARACTER
- */
- public static String getLoadNameFromLoadMetaDataDetails(
- List<LoadMetadataDetails> loadMetadataDetails) {
- StringBuilder builder = new StringBuilder();
- for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
- builder.append(CarbonCommonConstants.LOAD_FOLDER).append(loadMetadataDetail.getLoadName())
- .append(CarbonCommonConstants.HASH_SPC_CHARACTER);
- }
- String loadNames =
- builder.substring(0, builder.lastIndexOf(CarbonCommonConstants.HASH_SPC_CHARACTER));
- return loadNames;
- }
-
- /**
- * return the modOrDelTimesStamp TimeStamp Separated by HASH_SPC_CHARACTER
- */
- public static String getModificationOrDeletionTimesFromLoadMetadataDetails(
- List<LoadMetadataDetails> loadMetadataDetails) {
- StringBuilder builder = new StringBuilder();
- for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
- builder.append(loadMetadataDetail.getModificationOrdeletionTimesStamp())
- .append(CarbonCommonConstants.HASH_SPC_CHARACTER);
- }
- String modOrDelTimesStamp =
- builder.substring(0, builder.indexOf(CarbonCommonConstants.HASH_SPC_CHARACTER));
- return modOrDelTimesStamp;
- }
-
- /**
* This method will form the local data folder store location
*
* @param databaseName
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
index 136a8f3..fa5aff3 100644
--- a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
+++ b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
@@ -40,13 +40,9 @@ import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.carbon.datastore.block.TableBlockUniqueIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtilException;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.test.util.StoreCreator;
import junit.framework.TestCase;
-
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -77,7 +73,7 @@ public class BlockIndexStoreTest extends TestCase {
}
@Test public void testLoadAndGetTaskIdToSegmentsMapForSingleSegment()
- throws IOException, CarbonUtilException {
+ throws IOException {
String canonicalPath =
new File(this.getClass().getResource("/").getPath() + "/../../").getCanonicalPath();
File file = getPartFile();
@@ -94,7 +90,7 @@ public class BlockIndexStoreTest extends TestCase {
getTableBlockUniqueIdentifierList(Arrays.asList(new TableBlockInfo[] { info }), absoluteTableIdentifier);
List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockInfoList);
assertTrue(loadAndGetBlocks.size() == 1);
- } catch (CarbonUtilException e) {
+ } catch (Exception e) {
assertTrue(false);
}
List<String> segmentIds = new ArrayList<>();
@@ -161,7 +157,7 @@ public class BlockIndexStoreTest extends TestCase {
getTableBlockUniqueIdentifierList(tableBlockInfos, absoluteTableIdentifier);
List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockUniqueIdentifiers);
assertTrue(loadAndGetBlocks.size() == 5);
- } catch (CarbonUtilException e) {
+ } catch (Exception e) {
assertTrue(false);
}
List<String> segmentIds = new ArrayList<>();
@@ -233,7 +229,7 @@ public class BlockIndexStoreTest extends TestCase {
getTableBlockUniqueIdentifierList(tableBlockInfos, absoluteTableIdentifier);
List<AbstractIndex> loadAndGetBlocks = cache.getAll(blockUniqueIdentifierList);
assertTrue(loadAndGetBlocks.size() == 8);
- } catch (CarbonUtilException e) {
+ } catch (Exception e) {
assertTrue(false);
}
List<String> segmentIds = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
index 95f3d35..7e9feb6 100644
--- a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
@@ -82,7 +82,6 @@ import org.apache.carbondata.processing.dataprocessor.DataProcessTaskStatus;
import org.apache.carbondata.processing.dataprocessor.IDataProcessStatus;
import org.apache.carbondata.processing.graphgenerator.GraphGenerator;
import org.apache.carbondata.processing.graphgenerator.GraphGeneratorException;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
import com.google.gson.Gson;
import org.apache.hadoop.fs.Path;
@@ -377,8 +376,7 @@ public class StoreCreator {
DataGraphExecuter graphExecuter = new DataGraphExecuter(dataProcessTaskStatus);
graphExecuter
- .executeGraph(graphPath, new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN),
- info, "0", loadModel.getSchema());
+ .executeGraph(graphPath, info, loadModel.getSchema());
// LoadMetadataDetails[] loadDetails =
// CarbonUtil.readLoadMetadata(loadModel.schema.getCarbonTable().getMetaDataFilepath());
writeLoadMetadata(loadModel.schema, loadModel.getTableName(), loadModel.getTableName(),
[9/9] incubator-carbondata git commit: [CARBONDATA-568] clean up code
for carbon-core module This closes #472
Posted by ra...@apache.org.
[CARBONDATA-568] clean up code for carbon-core module This closes #472
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2ad623dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2ad623dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2ad623dc
Branch: refs/heads/master
Commit: 2ad623dcb4567286b436534f225235fec33aabab
Parents: 82072ee eaadc88
Author: ravipesala <ra...@gmail.com>
Authored: Wed Jan 4 19:04:28 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Jan 4 19:04:28 2017 +0530
----------------------------------------------------------------------
.../org/apache/carbondata/core/cache/Cache.java | 11 +-
.../AbstractColumnDictionaryInfo.java | 3 +-
.../dictionary/AbstractDictionaryCache.java | 105 +++--
.../dictionary/DictionaryBuilderException.java | 63 +++
.../dictionary/DoubleArrayTrieDictionary.java | 384 -------------------
.../dictionary/ForwardDictionaryCache.java | 18 +-
.../dictionary/ReverseDictionaryCache.java | 18 +-
.../core/carbon/CarbonDataLoadSchema.java | 25 +-
.../datastore/AbstractBlockIndexStoreCache.java | 14 +-
.../core/carbon/datastore/BlockIndexStore.java | 20 +-
.../core/carbon/datastore/DataRefNode.java | 16 +-
.../carbon/datastore/DataRefNodeFinder.java | 2 -
.../carbon/datastore/SegmentTaskIndexStore.java | 36 +-
.../carbon/datastore/block/AbstractIndex.java | 7 -
.../core/carbon/datastore/block/BlockIndex.java | 1 -
.../datastore/block/SegmentProperties.java | 5 +-
.../datastore/block/SegmentTaskIndex.java | 1 -
.../carbon/datastore/block/TableTaskInfo.java | 3 +-
.../chunk/DimensionColumnDataChunk.java | 3 -
.../reader/DimensionColumnChunkReader.java | 8 +-
.../chunk/reader/MeasureColumnChunkReader.java | 7 +-
.../reader/dimension/AbstractChunkReader.java | 1 -
...mpressedDimensionChunkFileBasedReaderV1.java | 5 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 7 +-
.../measure/AbstractMeasureChunkReader.java | 3 -
...CompressedMeasureChunkFileBasedReaderV1.java | 7 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 14 +-
.../exception/IndexBuilderException.java | 4 +-
.../impl/btree/AbstractBTreeBuilder.java | 3 +-
.../impl/btree/AbstractBTreeLeafNode.java | 13 +-
.../impl/btree/BTreeDataRefNodeFinder.java | 4 -
.../datastore/impl/btree/BTreeNonLeafNode.java | 5 -
.../impl/btree/BlockBTreeLeafNode.java | 1 -
.../impl/btree/BlockletBTreeLeafNode.java | 11 +-
.../compressor/ChunkCompressorMeta.java | 56 ---
.../blocklet/compressor/CompressionCodec.java | 31 --
.../metadata/blocklet/datachunk/DataChunk.java | 105 +----
.../metadata/blocklet/sort/SortState.java | 38 --
.../ThriftWrapperSchemaConverterImpl.java | 2 -
.../core/carbon/metadata/encoder/Encoding.java | 2 +-
.../metadata/schema/table/CarbonTable.java | 4 -
.../carbon/metadata/schema/table/TableInfo.java | 28 --
.../metadata/schema/table/TableSchema.java | 30 --
.../schema/table/column/CarbonMeasure.java | 19 -
.../schema/table/column/ColumnSchema.java | 21 -
.../core/carbon/path/CarbonTablePath.java | 1 -
.../QueryStatisticsRecorderDummy.java | 2 +-
.../core/compression/DoubleCompressor.java | 6 +-
.../core/constants/CarbonCommonConstants.java | 26 --
.../core/datastorage/store/FileHolder.java | 15 +-
.../core/datastorage/store/NodeKeyStore.java | 64 ----
.../datastorage/store/NodeMeasureDataStore.java | 6 -
.../store/columnar/ColumnarKeyStore.java | 47 ---
.../columnar/ColumnarKeyStoreDataHolder.java | 45 +--
.../store/columnar/ColumnarKeyStoreInfo.java | 262 -------------
.../columnar/ColumnarKeyStoreMetadata.java | 107 +-----
.../store/compression/WriterCompressModel.java | 3 +-
.../decimal/UnCompressByteArray.java | 2 +-
.../decimal/UnCompressMaxMinFloat.java | 2 +-
.../compression/none/UnCompressNoneDefault.java | 3 -
.../compression/none/UnCompressNoneLong.java | 6 +-
.../store/impl/DFSFileHolderImpl.java | 111 ++----
.../datastorage/store/impl/FileFactory.java | 68 ----
.../datastorage/store/impl/FileHolderImpl.java | 67 ++--
.../store/impl/MemoryMappedFileHolderImpl.java | 118 ------
...ractHeavyCompressedDoubleArrayDataStore.java | 8 +-
...yCompressedDoubleArrayDataInMemoryStore.java | 34 --
.../dictionary/generator/DictionaryWriter.java | 3 +-
.../IncrementalColumnDictionaryGenerator.java | 6 +-
.../generator/ServerDictionaryGenerator.java | 2 +-
.../generator/TableDictionaryGenerator.java | 9 +-
.../dictionary/server/DictionaryServer.java | 2 +-
.../core/keygenerator/KeyGenerator.java | 3 -
.../keygenerator/columnar/ColumnarSplitter.java | 18 -
.../MultiDimKeyVarLengthEquiSplitGenerator.java | 26 +-
...tiDimKeyVarLengthVariableSplitGenerator.java | 14 +-
.../TimeStampGranularityConstants.java | 4 +-
.../TimeStampGranularityTypeValue.java | 2 +-
.../mdkey/AbstractKeyGenerator.java | 2 +-
.../mdkey/MultiDimKeyVarLengthGenerator.java | 10 -
.../core/memory/HeapMemoryAllocator.java | 2 -
.../carbondata/core/metadata/BlockletInfo.java | 202 ----------
.../core/metadata/BlockletInfoColumnar.java | 49 ---
.../core/reader/CarbonDictionaryReaderImpl.java | 8 +-
.../core/reader/CarbonFooterReader.java | 5 +-
.../carbondata/core/reader/ThriftReader.java | 2 +-
.../CarbonDictionarySortIndexReader.java | 4 +-
.../core/service/ColumnUniqueIdService.java | 2 +-
.../core/service/DictionaryService.java | 10 +-
.../carbondata/core/unsafe/CarbonUnsafe.java | 2 +-
.../util/AbstractDataFileFooterConverter.java | 61 +--
.../carbondata/core/util/CarbonMergerUtil.java | 4 +-
.../core/util/CarbonMetadataUtil.java | 2 +-
.../carbondata/core/util/CarbonProperties.java | 16 -
.../core/util/CarbonTimeStatisticsFactory.java | 8 +-
.../apache/carbondata/core/util/CarbonUtil.java | 174 +++------
.../core/util/CarbonUtilException.java | 80 ----
.../carbondata/core/util/CompressionFinder.java | 2 +-
.../carbondata/core/util/DataTypeUtil.java | 2 +-
.../core/util/ValueCompressionUtil.java | 10 +-
.../core/writer/CarbonFooterWriter.java | 2 -
.../core/writer/CarbonIndexFileWriter.java | 1 -
.../core/writer/HierarchyValueWriterForCSV.java | 4 +-
.../CarbonDictionarySortIndexWriter.java | 7 +-
.../CarbonDictionarySortIndexWriterImpl.java | 5 +-
.../CarbonDictionarySortInfoPreparator.java | 4 +-
.../sortindex/CarbonDictionarySortModel.java | 12 +-
.../impl/DictionaryBasedResultCollector.java | 5 -
.../DictionaryBasedVectorResultCollector.java | 5 -
.../collector/impl/RawBasedResultCollector.java | 5 -
.../scan/complextypes/ArrayQueryType.java | 54 +--
.../scan/complextypes/ComplexQueryType.java | 12 +-
.../scan/complextypes/PrimitiveQueryType.java | 42 +-
.../scan/complextypes/StructQueryType.java | 60 +--
.../carbondata/scan/executor/QueryExecutor.java | 6 +-
.../exception/QueryExecutionException.java | 2 -
.../executor/impl/AbstractQueryExecutor.java | 47 +--
.../scan/executor/impl/DetailQueryExecutor.java | 3 +-
.../executor/impl/QueryExecutorProperties.java | 24 --
.../impl/VectorDetailQueryExecutor.java | 6 +-
.../scan/executor/infos/AggregatorInfo.java | 44 ---
.../scan/executor/infos/BlockExecutionInfo.java | 160 --------
.../scan/executor/infos/KeyStructureInfo.java | 19 -
.../scan/executor/infos/SortInfo.java | 125 ------
.../scan/executor/util/QueryUtil.java | 170 +-------
.../scan/expression/BinaryExpression.java | 16 -
.../carbondata/scan/expression/Expression.java | 1 -
.../conditional/ConditionalExpression.java | 6 -
.../exception/FilterIllegalMemberException.java | 4 -
.../exception/FilterUnsupportedException.java | 4 -
.../logical/BinaryLogicalExpression.java | 18 -
.../scan/filter/FilterExpressionProcessor.java | 22 +-
.../carbondata/scan/filter/FilterProcessor.java | 9 +-
.../carbondata/scan/filter/FilterUtil.java | 174 +++------
.../scan/filter/GenericQueryType.java | 22 +-
.../filter/executer/AndFilterExecuterImpl.java | 3 +-
.../ExcludeColGroupFilterExecuterImpl.java | 2 -
.../executer/ExcludeFilterExecuterImpl.java | 3 +-
.../scan/filter/executer/FilterExecuter.java | 4 +-
.../IncludeColGroupFilterExecuterImpl.java | 2 -
.../executer/IncludeFilterExecuterImpl.java | 8 +-
.../filter/executer/OrFilterExecuterImpl.java | 3 +-
.../executer/RowLevelFilterExecuterImpl.java | 86 ++---
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 7 +-
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 3 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 3 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 3 +-
.../RowLevelRangeTypeExecuterFacory.java | 9 +-
.../filter/resolver/AndFilterResolverImpl.java | 56 ---
.../resolver/ConditionalFilterResolverImpl.java | 39 +-
.../filter/resolver/FilterResolverIntf.java | 18 +-
.../resolver/LogicalFilterResolverImpl.java | 9 +-
.../resolver/RestructureFilterResolverImpl.java | 208 ----------
.../resolver/RowLevelFilterResolverImpl.java | 7 +-
.../RowLevelRangeFilterResolverImpl.java | 51 +--
.../DimColumnResolvedFilterInfo.java | 59 +--
.../MeasureColumnResolvedFilterInfo.java | 37 +-
.../visitable/ResolvedFilterInfoVisitable.java | 38 --
.../visitor/CustomTypeDictionaryVisitor.java | 13 +-
.../visitor/DictionaryColumnVisitor.java | 33 +-
.../visitor/NoDictionaryTypeVisitor.java | 7 +-
.../visitor/ResolvedFilterInfoVisitorIntf.java | 4 +-
.../carbondata/scan/model/CarbonQueryPlan.java | 83 ----
.../carbondata/scan/model/QueryColumn.java | 41 --
.../carbondata/scan/model/QueryModel.java | 184 +--------
.../carbondata/scan/model/SortOrderType.java | 57 ---
.../processor/AbstractDataBlockIterator.java | 21 +-
.../apache/carbondata/scan/result/Result.java | 70 ----
.../AbstractDetailQueryResultIterator.java | 1 -
.../iterator/DetailQueryResultIterator.java | 10 +-
.../scan/result/iterator/RawResultIterator.java | 2 +-
.../scan/result/vector/CarbonColumnVector.java | 24 +-
.../vector/MeasureDataVectorProcessor.java | 2 +-
.../scan/scanner/AbstractBlockletScanner.java | 8 +-
.../scan/scanner/BlockletScanner.java | 7 +-
.../scan/scanner/impl/FilterScanner.java | 29 +-
.../dictionary/ReverseDictionaryCacheTest.java | 5 +-
.../store/impl/DFSFileHolderImplUnitTest.java | 40 +-
.../store/impl/FileHolderImplUnitTest.java | 17 +-
.../MemoryMappedFileHolderImplUnitTest.java | 137 -------
.../datastore/SegmentTaskIndexStoreTest.java | 4 +-
.../datastore/block/SegmentPropertiesTest.java | 2 -
.../block/SegmentPropertiesTestUtil.java | 2 -
...ressedDimensionChunkFileBasedReaderTest.java | 5 +-
...mpressedMeasureChunkFileBasedReaderTest.java | 5 +-
.../carbon/metadata/CarbonMetadataTest.java | 1 -
.../ThriftWrapperSchemaConverterImplTest.java | 1 -
.../metadata/schema/table/CarbonTableTest.java | 1 -
.../table/CarbonTableWithComplexTypesTest.java | 1 -
.../ColumnarKeyStoreDataHolderUnitTest.java | 2 +-
.../carbondata/core/util/CarbonUtilTest.java | 56 +--
.../core/writer/CarbonFooterWriterTest.java | 4 +-
.../CarbonDictionarySortInfoPreparatorTest.java | 9 +-
.../impl/RawBasedResultCollectorTest.java | 1 -
.../carbondata/scan/filter/FilterUtilTest.java | 65 +---
.../scanner/impl/FilterScannerTest.java | 12 +-
.../carbondata/hadoop/CacheAccessClient.java | 12 +-
.../carbondata/hadoop/CarbonInputFormat.java | 41 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 35 +-
.../hadoop/readsupport/CarbonReadSupport.java | 8 +-
.../AbstractDictionaryDecodedReadSupport.java | 17 +-
.../hadoop/util/CarbonInputFormatUtil.java | 1 -
.../hadoop/test/util/StoreCreator.java | 4 +-
.../carbondata/spark/load/CarbonLoaderUtil.java | 14 +-
.../spark/merger/CarbonCompactionExecutor.java | 18 +-
.../spark/merger/CarbonCompactionUtil.java | 12 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 5 +-
.../execution/command/carbonTableSchema.scala | 1 -
.../readsupport/SparkRowReadSupportImpl.java | 3 +-
.../scala/org/apache/spark/sql/CarbonScan.scala | 10 -
.../spark/sql/SparkUnknownExpression.scala | 17 +-
.../readsupport/SparkRowReadSupportImpl.java | 4 +-
.../scala/org/apache/spark/sql/CarbonScan.scala | 3 -
.../processing/csvload/DataGraphExecuter.java | 128 ++-----
.../processing/csvload/GraphExecutionUtil.java | 2 +-
.../processing/datatypes/PrimitiveDataType.java | 9 +-
.../processing/mdkeygen/MDKeyGenStep.java | 5 +-
.../newflow/AbstractDataLoadProcessorStep.java | 5 +-
.../newflow/converter/RowConverter.java | 4 +-
.../impl/DictionaryFieldConverterImpl.java | 29 +-
.../converter/impl/FieldEncoderFactory.java | 4 +-
.../converter/impl/RowConverterImpl.java | 15 +-
.../impl/UnsafeParallelReadMergeSorterImpl.java | 2 +-
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 20 +-
.../merger/UnsafeIntermediateFileMerger.java | 3 +-
.../steps/DataConverterProcessorStepImpl.java | 4 +-
...ConverterProcessorWithBucketingStepImpl.java | 4 +-
.../steps/DataWriterProcessorStepImpl.java | 3 +-
.../newflow/steps/SortProcessorStepImpl.java | 3 +-
.../sortdata/IntermediateFileMerger.java | 3 +-
.../FileStoreSurrogateKeyGenForCSV.java | 55 +--
.../util/CarbonDataProcessorUtil.java | 40 +-
.../carbon/datastore/BlockIndexStoreTest.java | 12 +-
.../carbondata/test/util/StoreCreator.java | 4 +-
234 files changed, 946 insertions(+), 5250 deletions(-)
----------------------------------------------------------------------
[3/9] incubator-carbondata git commit: clean up core
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitable/ResolvedFilterInfoVisitable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitable/ResolvedFilterInfoVisitable.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitable/ResolvedFilterInfoVisitable.java
deleted file mode 100644
index bb71b96..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitable/ResolvedFilterInfoVisitable.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver.resolverinfo.visitable;
-
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor.ResolvedFilterInfoVisitorIntf;
-
-public interface ResolvedFilterInfoVisitable {
- /**
- * This visitable method will visit through the visitor classes which is passed as parameter
- * and based on different visitor the visitable filter instance will be resolved.
- *
- * @param visitor
- * @param metadata
- * @throws QueryExecutionException
- * @throws FilterUnsupportedException
- */
- void populateFilterInfoBasedOnColumnType(ResolvedFilterInfoVisitorIntf visitor,
- FilterResolverMetadata metadata) throws FilterUnsupportedException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index 95420be..9af4b71 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -22,9 +22,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
@@ -37,8 +34,6 @@ import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadat
import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorIntf {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CustomTypeDictionaryVisitor.class.getName());
/**
* This Visitor method is been used to resolve or populate the filter details
@@ -60,8 +55,8 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
} catch (FilterIllegalMemberException e) {
throw new FilterUnsupportedException(e);
}
- resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getTableIdentifier(),
- metadata.getColumnExpression(), evaluateResultListFinal, metadata.isIncludeFilter());
+ resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter());
if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
.getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
// Adding default surrogate key of null member inorder to not display the same while
@@ -74,8 +69,8 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
}
private DimColumnFilterInfo getDirectDictionaryValKeyMemberForFilter(
- AbsoluteTableIdentifier tableIdentifier, ColumnExpression columnExpression,
- List<String> evaluateResultListFinal, boolean isIncludeFilter) {
+ ColumnExpression columnExpression, List<String> evaluateResultListFinal,
+ boolean isIncludeFilter) {
List<Integer> surrogates = new ArrayList<Integer>(20);
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
index 6cb2348..532af60 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@ -18,13 +18,11 @@
*/
package org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor;
+import java.io.IOException;
import java.util.Collections;
import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.scan.filter.DimColumnFilterInfo;
@@ -33,8 +31,6 @@ import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadat
import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DictionaryColumnVisitor.class.getName());
/**
* This Visitor method is used to populate the visitableObj with direct dictionary filter details
@@ -44,10 +40,11 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
* @param metadata
* @throws FilterUnsupportedException,if exception occurs while evaluating
* filter models.
- * @throws QueryExecutionException
+ * @throws IOException
+ * @throws FilterUnsupportedException
*/
public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
- FilterResolverMetadata metadata) throws FilterUnsupportedException {
+ FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
DimColumnFilterInfo resolvedFilterObject = null;
List<String> evaluateResultListFinal;
try {
@@ -55,19 +52,15 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
} catch (FilterIllegalMemberException e) {
throw new FilterUnsupportedException(e);
}
- try {
- resolvedFilterObject = FilterUtil
- .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
- evaluateResultListFinal, metadata.isIncludeFilter());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
- }
- } catch (QueryExecutionException e) {
- throw new FilterUnsupportedException(e);
+ resolvedFilterObject = FilterUtil
+ .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
}
visitableObj.setFilterValues(resolvedFilterObject);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
index e7db0dc..035fd0d 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
@@ -20,8 +20,6 @@ package org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor;
import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
@@ -31,8 +29,6 @@ import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadat
import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(NoDictionaryTypeVisitor.class.getName());
/**
* Visitor Method will update the filter related details in visitableObj, For no dictionary
@@ -62,8 +58,7 @@ public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
throw new FilterUnsupportedException(e);
}
resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(metadata.getTableIdentifier(),
- metadata.getColumnExpression(), evaluateResultListFinal, metadata.isIncludeFilter());
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
visitableObj.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
index 1b2610d..2c46bde 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor;
+import java.io.IOException;
+
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -36,5 +38,5 @@ public interface ResolvedFilterInfoVisitorIntf {
* @throws FilterUnsupportedException
*/
void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
- FilterResolverMetadata metadata) throws FilterUnsupportedException;
+ FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/model/CarbonQueryPlan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/model/CarbonQueryPlan.java b/core/src/main/java/org/apache/carbondata/scan/model/CarbonQueryPlan.java
index 8ce732d..c77e59e 100644
--- a/core/src/main/java/org/apache/carbondata/scan/model/CarbonQueryPlan.java
+++ b/core/src/main/java/org/apache/carbondata/scan/model/CarbonQueryPlan.java
@@ -68,16 +68,6 @@ public class CarbonQueryPlan implements Serializable {
new ArrayList<QueryMeasure>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
/**
- * Limit
- */
- private int limit = -1;
-
- /**
- * If it is detail query, no need to aggregate in backend
- */
- private boolean detailQuery;
-
- /**
* expression
*/
private Expression expression;
@@ -88,33 +78,12 @@ public class CarbonQueryPlan implements Serializable {
private String queryId;
/**
- * outLocationPath
- */
- private String outLocationPath;
-
- /**
- * isCountStarQuery
- */
- private boolean isCountStartQuery;
-
- private List<QueryDimension> sortedDimensions;
-
- /**
* If it is raw detail query, no need to aggregate in backend. And it returns with dictionary data
* with out decoding.
*/
private boolean rawDetailQuery;
/**
- * Constructor created with table name.
- *
- * @param tableName
- */
- public CarbonQueryPlan(String tableName) {
- this.tableName = tableName;
- }
-
- /**
* Constructor created with database name and table name.
*
* @param databaseName
@@ -169,34 +138,6 @@ public class CarbonQueryPlan implements Serializable {
return tableName;
}
- /**
- * @return the limit
- */
- public int getLimit() {
- return limit;
- }
-
- /**
- * @param limit the limit to set
- */
- public void setLimit(int limit) {
- this.limit = limit;
- }
-
- /**
- * @return the detailQuery
- */
- public boolean isDetailQuery() {
- return detailQuery;
- }
-
- /**
- * @param detailQuery the detailQuery to set
- */
- public void setDetailQuery(boolean detailQuery) {
- this.detailQuery = detailQuery;
- }
-
public String getQueryId() {
return queryId;
}
@@ -205,30 +146,6 @@ public class CarbonQueryPlan implements Serializable {
this.queryId = queryId;
}
- public String getOutLocationPath() {
- return outLocationPath;
- }
-
- public void setOutLocationPath(String outLocationPath) {
- this.outLocationPath = outLocationPath;
- }
-
- public boolean isCountStarQuery() {
- return isCountStartQuery;
- }
-
- public void setCountStartQuery(boolean isCountStartQuery) {
- this.isCountStartQuery = isCountStartQuery;
- }
-
- public List<QueryDimension> getSortedDimemsions() {
- return sortedDimensions;
- }
-
- public void setSortedDimemsions(List<QueryDimension> dims) {
- this.sortedDimensions = dims;
- }
-
public boolean isRawDetailQuery() {
return rawDetailQuery;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/model/QueryColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/model/QueryColumn.java b/core/src/main/java/org/apache/carbondata/scan/model/QueryColumn.java
index 0aa5266..d026227 100644
--- a/core/src/main/java/org/apache/carbondata/scan/model/QueryColumn.java
+++ b/core/src/main/java/org/apache/carbondata/scan/model/QueryColumn.java
@@ -20,8 +20,6 @@ package org.apache.carbondata.scan.model;
import java.io.Serializable;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-
/**
* query column which will have information about column
*/
@@ -38,40 +36,15 @@ public class QueryColumn implements Serializable {
protected String columnName;
/**
- * sort order in which column output will be sorted default it will be none
- */
- private SortOrderType sortOrder = SortOrderType.NONE;
-
- /**
* query order in which result of the query will be send
*/
private int queryOrder;
- /**
- * aggregation function applied on column
- */
-
- private String aggregationFunction=CarbonCommonConstants.DUMMY;
-
public QueryColumn(String columnName) {
this.columnName = columnName;
}
/**
- * @return the sortOrder
- */
- public SortOrderType getSortOrder() {
- return sortOrder;
- }
-
- /**
- * @param sortOrder the sortOrder to set
- */
- public void setSortOrder(SortOrderType sortOrder) {
- this.sortOrder = sortOrder;
- }
-
- /**
* @return the columnName
*/
public String getColumnName() {
@@ -92,18 +65,4 @@ public class QueryColumn implements Serializable {
this.queryOrder = queryOrder;
}
- /**
- * @return the aggregationFunction
- */
- public String getAggregateFunction() {
- return aggregationFunction;
- }
-
- /**
- * @param aggregationFunction the aggregationFunction to set
- */
- public void setAggregateFunction(String aggregationFunction) {
- this.aggregationFunction = aggregationFunction;
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/scan/model/QueryModel.java
index 61332f5..89ac225 100644
--- a/core/src/main/java/org/apache/carbondata/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/scan/model/QueryModel.java
@@ -56,18 +56,10 @@ public class QueryModel implements Serializable {
*/
public transient Map<String, Dictionary> columnToDictionaryMapping;
/**
- * Number of records to keep in memory.
- */
- public int inMemoryRecordSize;
- /**
* list of dimension selected for in query
*/
private List<QueryDimension> queryDimension;
/**
- * list of dimension in which sorting is applied
- */
- private List<QueryDimension> sortDimension;
- /**
* list of measure selected in query
*/
private List<QueryMeasure> queryMeasures;
@@ -76,54 +68,24 @@ public class QueryModel implements Serializable {
*/
private String queryId;
/**
- * to check if it a aggregate table
- */
- private boolean isAggTable;
- /**
* filter tree
*/
private FilterResolverIntf filterExpressionResolverTree;
- /**
- * in case of limit query we need to know how many
- * records will passed from executor
- */
- private int limit;
/**
- * to check if it is a count star query , so processing will be different
- */
- private boolean isCountStarQuery;
- /**
- * to check whether aggregation is required during query execution
- */
- private boolean detailQuery;
- /**
* table block information in which query will be executed
*/
private List<TableBlockInfo> tableBlockInfos;
/**
- * sort in which dimension will be get sorted
- */
- private byte[] sortOrder;
- /**
* absolute table identifier
*/
private AbsoluteTableIdentifier absoluteTableIdentifier;
/**
- * in case of detail query with sort we are spilling to disk
- * to this location will be used to write the temp file in this location
- */
- private String queryTempLocation;
- /**
* To handle most of the computation in query engines like spark and hive, carbon should give
* raw detailed records to it.
*/
private boolean forcedDetailRawQuery;
/**
- * partition column list
- */
- private List<String> partitionColumns;
- /**
* table on which query will be executed
* TODO need to remove this ad pass only the path
* and carbon metadata will load the table from metadata file
@@ -145,9 +107,6 @@ public class QueryModel implements Serializable {
tableBlockInfos = new ArrayList<TableBlockInfo>();
queryDimension = new ArrayList<QueryDimension>();
queryMeasures = new ArrayList<QueryMeasure>();
- sortDimension = new ArrayList<QueryDimension>();
- sortOrder = new byte[0];
- partitionColumns = new ArrayList<String>();
invalidSegmentIds = new ArrayList<>();
}
@@ -159,11 +118,8 @@ public class QueryModel implements Serializable {
fillQueryModel(queryPlan, carbonTable, queryModel, factTableName);
- queryModel.setLimit(queryPlan.getLimit());
- queryModel.setDetailQuery(queryPlan.isDetailQuery());
queryModel.setForcedDetailRawQuery(queryPlan.isRawDetailQuery());
queryModel.setQueryId(queryPlan.getQueryId());
- queryModel.setQueryTempLocation(queryPlan.getOutLocationPath());
return queryModel;
}
@@ -171,36 +127,17 @@ public class QueryModel implements Serializable {
QueryModel queryModel, String factTableName) {
queryModel.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
queryModel.setQueryDimension(queryPlan.getDimensions());
- fillSortInfoInModel(queryModel, queryPlan.getSortedDimemsions());
queryModel.setQueryMeasures(queryPlan.getMeasures());
if (null != queryPlan.getFilterExpression()) {
processFilterExpression(queryPlan.getFilterExpression(),
carbonTable.getDimensionByTableName(factTableName),
carbonTable.getMeasureByTableName(factTableName));
}
- queryModel.setCountStarQuery(queryPlan.isCountStarQuery());
//TODO need to remove this code, and executor will load the table
// from file metadata
queryModel.setTable(carbonTable);
}
- private static void fillSortInfoInModel(QueryModel executorModel,
- List<QueryDimension> sortedDims) {
- if (null != sortedDims) {
- byte[] sortOrderByteArray = new byte[sortedDims.size()];
- int i = 0;
- for (QueryColumn mdim : sortedDims) {
- sortOrderByteArray[i++] = (byte) mdim.getSortOrder().ordinal();
- }
- executorModel.setSortOrder(sortOrderByteArray);
- executorModel.setSortDimension(sortedDims);
- } else {
- executorModel.setSortOrder(new byte[0]);
- executorModel.setSortDimension(new ArrayList<QueryDimension>(0));
- }
-
- }
-
public static void processFilterExpression(Expression filterExpression,
List<CarbonDimension> dimensions, List<CarbonMeasure> measures) {
if (null != filterExpression) {
@@ -209,13 +146,11 @@ public class QueryModel implements Serializable {
List<ColumnExpression> listOfCol =
((ConditionalExpression) filterExpression).getColumnList();
for (ColumnExpression expression : listOfCol) {
- setDimAndMsrColumnNode(dimensions, measures, (ColumnExpression) expression);
+ setDimAndMsrColumnNode(dimensions, measures, expression);
}
-
}
}
for (Expression expression : filterExpression.getChildren()) {
-
if (expression instanceof ColumnExpression) {
setDimAndMsrColumnNode(dimensions, measures, (ColumnExpression) expression);
} else if (expression instanceof UnknownExpression) {
@@ -316,59 +251,6 @@ public class QueryModel implements Serializable {
}
/**
- * @return the isAggTable
- */
- public boolean isAggTable() {
- return isAggTable;
- }
-
- /**
- * @param isAggTable the isAggTable to set
- */
- public void setAggTable(boolean isAggTable) {
- this.isAggTable = isAggTable;
- }
-
- /**
- * @return the limit
- */
- public int getLimit() {
- return limit;
- }
-
- /**
- * @param limit the limit to set
- */
- public void setLimit(int limit) {
- this.limit = limit;
- }
-
- /**
- * @return the isCountStarQuery
- */
- public boolean isCountStarQuery() {
- return isCountStarQuery;
- }
-
- /**
- * @param isCountStarQuery the isCountStarQuery to set
- */
- public void setCountStarQuery(boolean isCountStarQuery) {
- this.isCountStarQuery = isCountStarQuery;
- }
-
- /**
- * @return the isdetailQuery
- */
- public boolean isDetailQuery() {
- return detailQuery;
- }
-
- public void setDetailQuery(boolean detailQuery) {
- this.detailQuery = detailQuery;
- }
-
- /**
* @return the tableBlockInfos
*/
public List<TableBlockInfo> getTableBlockInfos() {
@@ -383,48 +265,6 @@ public class QueryModel implements Serializable {
}
/**
- * @return the queryTempLocation
- */
- public String getQueryTempLocation() {
- return queryTempLocation;
- }
-
- /**
- * @param queryTempLocation the queryTempLocation to set
- */
- public void setQueryTempLocation(String queryTempLocation) {
- this.queryTempLocation = queryTempLocation;
- }
-
- /**
- * @return the sortOrder
- */
- public byte[] getSortOrder() {
- return sortOrder;
- }
-
- /**
- * @param sortOrder the sortOrder to set
- */
- public void setSortOrder(byte[] sortOrder) {
- this.sortOrder = sortOrder;
- }
-
- /**
- * @return the sortDimension
- */
- public List<QueryDimension> getSortDimension() {
- return sortDimension;
- }
-
- /**
- * @param sortDimension the sortDimension to set
- */
- public void setSortDimension(List<QueryDimension> sortDimension) {
- this.sortDimension = sortDimension;
- }
-
- /**
* @return the filterEvaluatorTree
*/
public FilterResolverIntf getFilterExpressionResolverTree() {
@@ -450,20 +290,6 @@ public class QueryModel implements Serializable {
}
/**
- * @return the partitionColumns
- */
- public List<String> getPartitionColumns() {
- return partitionColumns;
- }
-
- /**
- * @param partitionColumns the partitionColumns to set
- */
- public void setPartitionColumns(List<String> partitionColumns) {
- this.partitionColumns = partitionColumns;
- }
-
- /**
* @return the table
*/
public CarbonTable getTable() {
@@ -499,14 +325,6 @@ public class QueryModel implements Serializable {
this.columnToDictionaryMapping = columnToDictionaryMapping;
}
- public int getInMemoryRecordSize() {
- return inMemoryRecordSize;
- }
-
- public void setInMemoryRecordSize(int inMemoryRecordSize) {
- this.inMemoryRecordSize = inMemoryRecordSize;
- }
-
public QueryStatisticsRecorder getStatisticsRecorder() {
return statisticsRecorder;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/model/SortOrderType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/model/SortOrderType.java b/core/src/main/java/org/apache/carbondata/scan/model/SortOrderType.java
deleted file mode 100644
index ba725b7..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/model/SortOrderType.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.model;
-
-/**
- * enum for sorting the columns
- */
-public enum SortOrderType {
-
- /**
- * Ascending order
- */
- ASC(0),
-
- /**
- * Descending order.
- */
- DSC(1),
-
- /**
- * No order mentioned
- */
- NONE(-1);
- /**
- * Order type in numeric
- */
- private int orderType;
-
- SortOrderType(int orderType) {
- this.orderType = orderType;
- }
-
- /**
- * Order type in number
- *
- * @return orderType int
- */
- public int getOrderType() {
- return orderType;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
index 95d5836..4fc74b9 100644
--- a/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
@@ -18,11 +18,10 @@
*/
package org.apache.carbondata.scan.processor;
+import java.io.IOException;
import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.carbon.datastore.DataRefNode;
import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsModel;
import org.apache.carbondata.core.datastorage.store.FileHolder;
@@ -30,8 +29,8 @@ import org.apache.carbondata.scan.collector.ScannedResultCollector;
import org.apache.carbondata.scan.collector.impl.DictionaryBasedResultCollector;
import org.apache.carbondata.scan.collector.impl.DictionaryBasedVectorResultCollector;
import org.apache.carbondata.scan.collector.impl.RawBasedResultCollector;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.scan.result.AbstractScannedResult;
import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.scan.scanner.BlockletScanner;
@@ -44,19 +43,12 @@ import org.apache.carbondata.scan.scanner.impl.NonFilterScanner;
*/
public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Object[]>> {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractDataBlockIterator.class.getName());
/**
* iterator which will be used to iterate over data blocks
*/
protected CarbonIterator<DataRefNode> dataBlockIterator;
/**
- * execution details
- */
- protected BlockExecutionInfo blockExecutionInfo;
-
- /**
* result collector which will be used to aggregate the scanned result
*/
protected ScannedResultCollector scannerResultAggregator;
@@ -79,11 +71,8 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
protected AbstractScannedResult scannedResult;
- QueryStatisticsModel queryStatisticsModel;
-
public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo,
FileHolder fileReader, int batchSize, QueryStatisticsModel queryStatisticsModel) {
- this.blockExecutionInfo = blockExecutionInfo;
dataBlockIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
blockExecutionInfo.getNumberOfBlockToScan());
blocksChunkHolder = new BlocksChunkHolder(blockExecutionInfo.getTotalNumberDimensionBlock(),
@@ -106,7 +95,6 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
new DictionaryBasedResultCollector(blockExecutionInfo);
}
this.batchSize = batchSize;
- this.queryStatisticsModel = queryStatisticsModel;
}
public boolean hasNext() {
@@ -131,12 +119,13 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
}
return false;
}
- } catch (QueryExecutionException ex) {
+ } catch (IOException | FilterUnsupportedException ex) {
throw new RuntimeException(ex);
}
}
- private AbstractScannedResult getNextScannedResult() throws QueryExecutionException {
+ private AbstractScannedResult getNextScannedResult()
+ throws IOException, FilterUnsupportedException {
if (dataBlockIterator.hasNext()) {
blocksChunkHolder.setDataBlock(dataBlockIterator.next());
blocksChunkHolder.reset();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/result/Result.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/Result.java b/core/src/main/java/org/apache/carbondata/scan/result/Result.java
deleted file mode 100644
index 618af73..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/Result.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.result;
-
-import org.apache.carbondata.scan.wrappers.ByteArrayWrapper;
-
-/**
- * Result interface for storing the result
- */
-public interface Result<K, V> {
- /**
- * Below method will be used to
- * add the sccaed result
- *
- * @param result
- */
- void addScannedResult(K result);
-
- /**
- * Returns {@code true} if the iteration has more elements.
- *
- * @return {@code true} if the iteration has more elements
- */
- boolean hasNext();
-
- /**
- * Below method will return the result key
- *
- * @return key
- */
- ByteArrayWrapper getKey();
-
- /**
- * Below code will return the result value
- *
- * @return value
- */
- V[] getValue();
-
- void merge(Result<K, V> otherResult);
-
- /**
- * Below method will be used to get the result
- *
- * @return
- */
- K getResult();
-
- /**
- * @return size of the result
- */
- int size();
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
index f6b6ba1..5a0d3c1 100644
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -173,7 +173,6 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
QueryStatistic queryStatisticTotalBlocklet = new QueryStatistic();
queryStatisticsModel.getStatisticsTypeAndObjMap()
.put(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, queryStatisticTotalBlocklet);
- QueryStatistic queryStatisticScanBlocklet = new QueryStatistic();
QueryStatistic queryStatisticValidScanBlocklet = new QueryStatistic();
queryStatisticsModel.getStatisticsTypeAndObjMap()
.put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatisticValidScanBlocklet);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/scan/result/iterator/DetailQueryResultIterator.java
index 7d944d6..9afe0f5 100644
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/scan/result/iterator/DetailQueryResultIterator.java
@@ -23,7 +23,6 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.scan.model.QueryModel;
import org.apache.carbondata.scan.result.BatchResult;
@@ -60,15 +59,18 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
}
totalScanTime += System.currentTimeMillis() - startTime;
} catch (Exception ex) {
- fileReader.finish();
- throw new RuntimeException(ex);
+ try {
+ fileReader.finish();
+ } finally {
+ throw new RuntimeException(ex);
+ }
}
return result;
}
private Future<BatchResult> execute() {
return execService.submit(new Callable<BatchResult>() {
- @Override public BatchResult call() throws QueryExecutionException {
+ @Override public BatchResult call() {
BatchResult batchResult = new BatchResult();
synchronized (lock) {
updateDataBlockIterator();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/result/iterator/RawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/RawResultIterator.java b/core/src/main/java/org/apache/carbondata/scan/result/iterator/RawResultIterator.java
index 4b20627..b37a989 100644
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/RawResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/scan/result/iterator/RawResultIterator.java
@@ -134,7 +134,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
if(hasNext())
{
Object[] rawRow = batch.getRawRow(counter);
- currentConveretedRawRow = convertRow(rawRow);;
+ currentConveretedRawRow = convertRow(rawRow);
return currentConveretedRawRow;
}
else
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java b/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
index d89f52f..0af1857 100644
--- a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
+++ b/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
@@ -20,28 +20,28 @@ import org.apache.spark.sql.types.Decimal;
public interface CarbonColumnVector {
- public void putShort(int rowId, short value);
+ void putShort(int rowId, short value);
- public void putInt(int rowId, int value);
+ void putInt(int rowId, int value);
- public void putLong(int rowId, long value);
+ void putLong(int rowId, long value);
- public void putDecimal(int rowId, Decimal value, int precision);
+ void putDecimal(int rowId, Decimal value, int precision);
- public void putDouble(int rowId, double value);
+ void putDouble(int rowId, double value);
- public void putBytes(int rowId, byte[] value);
+ void putBytes(int rowId, byte[] value);
- public void putBytes(int rowId, int offset, int length, byte[] value);
+ void putBytes(int rowId, int offset, int length, byte[] value);
- public void putNull(int rowId);
+ void putNull(int rowId);
- public boolean isNull(int rowId);
+ boolean isNull(int rowId);
- public void putObject(int rowId, Object obj);
+ void putObject(int rowId, Object obj);
- public Object getData(int rowId);
+ Object getData(int rowId);
- public void reset();
+ void reset();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/result/vector/MeasureDataVectorProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/scan/result/vector/MeasureDataVectorProcessor.java
index 9e455e2..c7060a3 100644
--- a/core/src/main/java/org/apache/carbondata/scan/result/vector/MeasureDataVectorProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/result/vector/MeasureDataVectorProcessor.java
@@ -28,7 +28,7 @@ import org.apache.spark.sql.types.Decimal;
public class MeasureDataVectorProcessor {
- public static interface MeasureVectorFiller {
+ public interface MeasureVectorFiller {
void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
index cb6a4ad..84c0cb4 100644
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
@@ -18,11 +18,13 @@
*/
package org.apache.carbondata.scan.scanner;
+import java.io.IOException;
+
import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic;
import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsConstants;
import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsModel;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.scan.result.AbstractScannedResult;
@@ -48,12 +50,12 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
}
@Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws QueryExecutionException {
+ throws IOException, FilterUnsupportedException {
fillKeyValue(blocksChunkHolder);
return scannedResult;
}
- protected void fillKeyValue(BlocksChunkHolder blocksChunkHolder) {
+ protected void fillKeyValue(BlocksChunkHolder blocksChunkHolder) throws IOException {
QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
.get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/scanner/BlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/BlockletScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/BlockletScanner.java
index 0337197..39a20b2 100644
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/BlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/scan/scanner/BlockletScanner.java
@@ -18,7 +18,9 @@
*/
package org.apache.carbondata.scan.scanner;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
+import java.io.IOException;
+
+import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.scan.result.AbstractScannedResult;
@@ -34,8 +36,7 @@ public interface BlockletScanner {
* @param blocksChunkHolder block chunk which holds the block data
* @return scannerResult
* result after processing
- * @throws QueryExecutionException
*/
AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws QueryExecutionException;
+ throws IOException, FilterUnsupportedException;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
index ac71100..5ff61d4 100644
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
@@ -16,8 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.carbondata.scan.scanner.impl;
+import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
@@ -28,7 +30,6 @@ import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsModel;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.store.FileHolder;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.scan.filter.executer.FilterExecuter;
@@ -81,16 +82,11 @@ public class FilterScanner extends AbstractBlockletScanner {
* Below method will be used to process the block
*
* @param blocksChunkHolder block chunk holder which holds the data
- * @throws QueryExecutionException
* @throws FilterUnsupportedException
*/
@Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws QueryExecutionException {
- try {
- fillScannedResult(blocksChunkHolder);
- } catch (FilterUnsupportedException e) {
- throw new QueryExecutionException(e.getMessage());
- }
+ throws IOException, FilterUnsupportedException {
+ fillScannedResult(blocksChunkHolder);
return scannedResult;
}
@@ -111,8 +107,7 @@ public class FilterScanner extends AbstractBlockletScanner {
* @throws FilterUnsupportedException
*/
private void fillScannedResult(BlocksChunkHolder blocksChunkHolder)
- throws FilterUnsupportedException {
-
+ throws FilterUnsupportedException, IOException {
scannedResult.reset();
QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
.get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
@@ -167,10 +162,9 @@ public class FilterScanner extends AbstractBlockletScanner {
}
}
for (int i = 0; i < allSelectedDimensionBlocksIndexes.length; i++) {
- for (int j = allSelectedDimensionBlocksIndexes[i][0];
- j <= allSelectedDimensionBlocksIndexes[i][1]; j++) {
- dimensionColumnDataChunk[j] = projectionListDimensionChunk[j];
- }
+ System.arraycopy(projectionListDimensionChunk, allSelectedDimensionBlocksIndexes[i][0],
+ dimensionColumnDataChunk, allSelectedDimensionBlocksIndexes[i][0],
+ allSelectedDimensionBlocksIndexes[i][1] + 1 - allSelectedDimensionBlocksIndexes[i][0]);
}
MeasureColumnDataChunk[] measureColumnDataChunk =
new MeasureColumnDataChunk[blockExecutionInfo.getTotalNumberOfMeasureBlock()];
@@ -185,10 +179,9 @@ public class FilterScanner extends AbstractBlockletScanner {
}
}
for (int i = 0; i < allSelectedMeasureBlocksIndexes.length; i++) {
- for (int j = allSelectedMeasureBlocksIndexes[i][0];
- j <= allSelectedMeasureBlocksIndexes[i][1]; j++) {
- measureColumnDataChunk[j] = projectionListMeasureChunk[j];
- }
+ System.arraycopy(projectionListMeasureChunk, allSelectedMeasureBlocksIndexes[i][0],
+ measureColumnDataChunk, allSelectedMeasureBlocksIndexes[i][0],
+ allSelectedMeasureBlocksIndexes[i][1] + 1 - allSelectedMeasureBlocksIndexes[i][0]);
}
scannedResult.setDimensionChunks(dimensionColumnDataChunk);
scannedResult.setIndexes(indexes);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
index c526f45..49b5a7b 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -39,7 +40,7 @@ import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtilException;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -182,7 +183,7 @@ public class ReverseDictionaryCacheTest extends AbstractDictionaryCacheTest {
reverseDictionary = (Dictionary) reverseDictionaryCache.get(dictionaryColumnUniqueIdentifier);
fail("not throwing exception");
} catch (Exception e) {
- assertTrue(e instanceof CarbonUtilException);
+ assertTrue(e instanceof IOException);
}
assertEquals(null, reverseDictionary);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
index 38d9f44..3b3ab9f 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
@@ -76,68 +76,70 @@ public class DFSFileHolderImplUnitTest {
fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
}
- @AfterClass public static void tearDown() {
+ @AfterClass public static void tearDown() throws IOException {
file.delete();
fileWithEmptyContent.delete();
dfsFileHolder.finish();
}
- @Test public void testReadByteArray() {
+ @Test public void testReadByteArray() throws IOException {
byte[] result = dfsFileHolder.readByteArray(fileName, 1);
byte[] expected_result = new byte[] { 72 };
assertThat(result, is(equalTo(expected_result)));
}
- @Test public void testReadByteArrayWithFilePath() {
+ @Test public void testReadByteArrayWithFilePath() throws IOException {
byte[] result = dfsFileHolder.readByteArray(fileName, 2L, 2);
byte[] expected_result = { 108, 108 };
assertThat(result, is(equalTo(expected_result)));
}
- @Test public void testReadLong() {
+ @Test public void testReadLong() throws IOException {
long actualResult = dfsFileHolder.readLong(fileName, 1L);
long expectedResult = 7308335519855243122L;
assertThat(actualResult, is(equalTo(expectedResult)));
}
- @Test public void testReadLongForIoException() throws IOException {
+ @Test(expected = IOException.class)
+ public void testReadLongForIoException() throws IOException {
dfsFileHolder.readLong(fileNameWithEmptyContent, 1L);
-
}
- @Test public void testReadIntForIoException() throws IOException {
+ @Test(expected = IOException.class)
+ public void testReadIntForIoException() throws IOException{
dfsFileHolder.readInt(fileNameWithEmptyContent, 1L);
}
- @Test public void testReadInt() {
+ @Test public void testReadInt() throws IOException {
int actualResult = dfsFileHolder.readInt(fileName, 1L);
int expectedResult = 1701604463;
assertThat(actualResult, is(equalTo(expectedResult)));
}
- @Test public void testReadIntWithFileName() {
+ @Test public void testReadIntWithFileName() throws IOException {
int actualResult = dfsFileHolder.readInt(fileName);
int expectedResult = 1701604463;
assertThat(actualResult, is(equalTo(expectedResult)));
}
- @Test public void testReadIntWithFileNameForIOException() {
+ @Test(expected = IOException.class)
+ public void testReadIntWithFileNameForIOException() throws IOException {
dfsFileHolder.readInt(fileNameWithEmptyContent);
-
}
- @Test public void testDouble() {
+ @Test public void testDouble() throws IOException {
double actualResult = dfsFileHolder.readDouble(fileName, 1L);
double expectedResult = 7.3083355198552433E18;
assertThat(actualResult, is(equalTo(expectedResult)));
}
- @Test public void testDoubleForIoException() throws IOException {
+ @Test(expected = IOException.class)
+ public void testDoubleForIoException() throws IOException {
dfsFileHolder.readDouble(fileNameWithEmptyContent, 1L);
-
}
- @Test public void testDoubleForIoExceptionwithUpdateCache() throws Exception {
+ @Test
+ public void testDoubleForIoExceptionwithUpdateCache() throws IOException {
new MockUp<FileSystem>() {
@SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
throws IOException {
@@ -145,13 +147,7 @@ public class DFSFileHolderImplUnitTest {
}
};
- String expected = null;
- try {
- dfsFileHolder.readDouble(fileName, 1L);
- } catch (Exception e) {
- assertNull(e.getMessage());
- }
-
+ dfsFileHolder.readDouble(fileName, 1L);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
index 127f2e0..c14dff9 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
@@ -78,25 +78,25 @@ public class FileHolderImplUnitTest {
fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
}
- @AfterClass public static void tearDown() {
+ @AfterClass public static void tearDown() throws IOException {
file.delete();
fileWithEmptyContent.delete();
fileHolder.finish();
}
- @Test public void testReadByteArray() {
+ @Test public void testReadByteArray() throws IOException {
byte[] result = fileHolder.readByteArray(fileName, 1);
byte[] expected_result = new byte[] { 72 };
assertThat(result, is(equalTo(expected_result)));
}
- @Test public void testReadByteArrayWithFilePath() {
+ @Test public void testReadByteArrayWithFilePath() throws IOException {
byte[] result = fileHolder.readByteArray(fileName, 2L, 2);
byte[] expected_result = { 108, 108 };
assertThat(result, is(equalTo(expected_result)));
}
- @Test public void testReadLong() {
+ @Test public void testReadLong() throws IOException {
long actualResult = fileHolder.readLong(fileName, 1L);
long expectedResult = 7308335519855243122L;
assertThat(actualResult, is(equalTo(expectedResult)));
@@ -104,31 +104,30 @@ public class FileHolderImplUnitTest {
@Test public void testReadLongForIoException() throws IOException {
fileHolder.readLong(fileNameWithEmptyContent, 1L);
-
}
@Test public void testReadIntForIoException() throws IOException {
fileHolder.readInt(fileNameWithEmptyContent, 1L);
}
- @Test public void testReadInt() {
+ @Test public void testReadInt() throws IOException {
int actualResult = fileHolder.readInt(fileName, 1L);
int expectedResult = 1701604463;
assertThat(actualResult, is(equalTo(expectedResult)));
}
- @Test public void testReadIntWithFileName() {
+ @Test public void testReadIntWithFileName() throws IOException {
int actualResult = fileHolder.readInt(fileName);
int expectedResult = 1701604463;
assertThat(actualResult, is(equalTo(expectedResult)));
}
- @Test public void testReadIntWithFileNameForIOException() {
+ @Test public void testReadIntWithFileNameForIOException() throws IOException {
fileHolder.readInt(fileNameWithEmptyContent);
}
- @Test public void testDouble() {
+ @Test public void testDouble() throws IOException {
double actualResult = fileHolder.readDouble(fileName, 1L);
double expectedResult = 7.3083355198552433E18;
assertThat(actualResult, is(equalTo(expectedResult)));
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/MemoryMappedFileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/MemoryMappedFileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/MemoryMappedFileHolderImplUnitTest.java
deleted file mode 100644
index 29ca475..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/MemoryMappedFileHolderImplUnitTest.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import org.apache.carbondata.core.datastorage.store.impl.MemoryMappedFileHolderImpl;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-
-public class MemoryMappedFileHolderImplUnitTest {
-
- private static MemoryMappedFileHolderImpl memoryMappedFileHolder = null;
- private static MemoryMappedFileHolderImpl memoryMappedFileHolderWithCapacity = null;
- private static String fileName = null;
- private static String fileWithEmptyContentName = null;
- private static File file = null;
- private static File fileWithEmptyContent = null;
-
- @BeforeClass public static void setup() {
- memoryMappedFileHolder = new MemoryMappedFileHolderImpl();
- memoryMappedFileHolderWithCapacity = new MemoryMappedFileHolderImpl(50);
- file = new File("Test.carbondata");
- fileWithEmptyContent = new File("TestEXception.carbondata");
-
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- if (!fileWithEmptyContent.exists()) try {
- fileWithEmptyContent.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream of = new FileOutputStream(file, true);
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
- br.write("Hello World");
- br.close();
- } catch (Exception e) {
- e.getMessage();
- }
- fileName = file.getAbsolutePath();
- fileWithEmptyContentName = fileWithEmptyContent.getAbsolutePath();
-
- }
-
- @AfterClass public static void tearDown() {
- file.delete();
- fileWithEmptyContent.delete();
- memoryMappedFileHolder.finish();
- }
-
- @Test public void testReadByteArray() {
- byte[] result = memoryMappedFileHolder.readByteArray(fileName, 1);
- byte[] expected_result = new byte[] { 72 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadByteArrayWithFilePath() {
-
- final byte[] result = memoryMappedFileHolder.readByteArray(fileName, 0L, 6);
- final byte[] expected_result = { 32, 87, 111, 114, 108, 100 };
-
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadLong() {
-
- long actualResult = memoryMappedFileHolderWithCapacity.readLong(fileName, 0L);
- long expectedResult = 5216694956355245935L;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadInt() {
- memoryMappedFileHolder = new MemoryMappedFileHolderImpl();
-
- int actualResult = memoryMappedFileHolder.readInt(fileName, 0L);
- int expectedResult = 1214606444;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileName() {
-
- int actualResult = memoryMappedFileHolder.readInt(fileName);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testDouble() {
- memoryMappedFileHolder = new MemoryMappedFileHolderImpl();
-
- double actualResult = memoryMappedFileHolder.readDouble(fileName, 0L);
- double expectedResult = 5.2166949563552461E18;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testDoubleForIoExceptionwithUpdateCache() throws Exception {
-
- String expected = null;
- try {
- memoryMappedFileHolder.readDouble(fileName, 0L);
- } catch (Exception e) {
- assertEquals(e.getMessage(), expected);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStoreTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStoreTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStoreTest.java
index 8adb8bb..17e39f3 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStoreTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStoreTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.core.carbon.datastore;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -39,7 +40,6 @@ import org.apache.carbondata.core.carbon.metadata.blocklet.SegmentInfo;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.carbon.path.CarbonTablePath;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import mockit.Mock;
import mockit.MockUp;
@@ -87,7 +87,7 @@ public class SegmentTaskIndexStoreTest {
return footerList;
}
- @Test public void loadAndGetTaskIdToSegmentsMap() throws CarbonUtilException {
+ @Test public void loadAndGetTaskIdToSegmentsMap() throws IOException {
new MockUp<CarbonTablePath.DataFileUtil>() {
@Mock String getTaskNo(String carbonDataFileName) {
return "100";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTest.java
index 52e84ee..72d47df 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTest.java
@@ -325,7 +325,6 @@ public class SegmentPropertiesTest extends TestCase {
private ColumnSchema getMeasureColumn() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setAggregateFunction("SUM");
dimColumn.setColumnName("IMEI_COUNT");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataType.STRING);
@@ -338,7 +337,6 @@ public class SegmentPropertiesTest extends TestCase {
private ColumnSchema getMeasureColumn1() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setAggregateFunction("SUM");
dimColumn.setColumnName("IMEI_COUNT1");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataType.STRING);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTestUtil.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTestUtil.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTestUtil.java
index 807b11b..ed1068a 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTestUtil.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/block/SegmentPropertiesTestUtil.java
@@ -208,7 +208,6 @@ public class SegmentPropertiesTestUtil {
public static ColumnSchema getMeasureColumn() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setAggregateFunction("SUM");
dimColumn.setColumnName("IMEI_COUNT");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataType.STRING);
@@ -221,7 +220,6 @@ public class SegmentPropertiesTestUtil {
public static ColumnSchema getMeasureColumn1() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setAggregateFunction("SUM");
dimColumn.setColumnName("IMEI_COUNT1");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataType.STRING);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java
index 57379c1..5a043b0 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java
@@ -19,6 +19,7 @@
package org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -57,7 +58,7 @@ public class CompressedDimensionChunkFileBasedReaderTest {
new CompressedDimensionChunkFileBasedReaderV1(info, eachColumnBlockSize, "filePath");
}
- @Test public void readDimensionChunksTest() {
+ @Test public void readDimensionChunksTest() throws IOException {
FileHolder fileHolder = new MockUp<FileHolder>() {
@Mock public byte[] readByteArray(String filePath, long offset, int length) {
byte mockedValue[] = { 1, 5, 4, 8, 7 };
@@ -101,7 +102,7 @@ public class CompressedDimensionChunkFileBasedReaderTest {
}
}
- @Test public void readDimensionChunksTestForIfStatement() {
+ @Test public void readDimensionChunksTestForIfStatement() throws IOException {
FileHolder fileHolder = new MockUp<FileHolder>() {
@Mock public byte[] readByteArray(String filePath, long offset, int length) {
byte mockedValue[] = { 1, 5, 4, 8, 7 };
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
index c2a5d5d..0080690 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
@@ -2,6 +2,7 @@ package org.apache.carbondata.core.carbon.datastore.chunk.reader.measure;
import static junit.framework.TestCase.assertEquals;
+import java.io.IOException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
@@ -58,7 +59,7 @@ public class CompressedMeasureChunkFileBasedReaderTest {
new CompressedMeasureChunkFileBasedReaderV1(info, "filePath");
}
- @Test public void readMeasureChunkTest() {
+ @Test public void readMeasureChunkTest() throws IOException {
FileHolder fileHolder = new MockUp<FileHolder>() {
@Mock public byte[] readByteArray(String filePath, long offset, int length) {
dataHolder[0] = new CarbonWriteDataHolder();
@@ -80,7 +81,7 @@ public class CompressedMeasureChunkFileBasedReaderTest {
}
- @Test public void readMeasureChunksTest() {
+ @Test public void readMeasureChunksTest() throws IOException {
FileHolder fileHolder = new MockUp<FileHolder>() {
@Mock public byte[] readByteArray(String filePath, long offset, int length) {
dataHolder[0] = new CarbonWriteDataHolder();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/CarbonMetadataTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/CarbonMetadataTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/metadata/CarbonMetadataTest.java
index 85ae9b4..9c74a4a 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/CarbonMetadataTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/metadata/CarbonMetadataTest.java
@@ -134,7 +134,6 @@ public class CarbonMetadataTest {
private static ColumnSchema getColumnarMeasureColumn() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setAggregateFunction("SUM");
dimColumn.setColumnName("IMEI_COUNT");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataType.STRING);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index 4022ff4..4af55c0 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -1555,7 +1555,6 @@ long time =1112745600000L;
wrapperColumnSchema.setColumnGroup(1);
wrapperColumnSchema.setScale(1);
wrapperColumnSchema.setDefaultValue(new byte[] { 1, 2 });
- wrapperColumnSchema.setAggregateFunction("");
wrapperColumnSchema.setColumnProperties(columnPropertyMap);
wrapperColumnSchema.setInvisible(true);
wrapperColumnSchema.setColumnReferenceId("1");
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableTest.java
index ff74e14..ca5def5 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableTest.java
@@ -88,7 +88,6 @@ public class CarbonTableTest extends TestCase {
private ColumnSchema getColumnarMeasureColumn() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setAggregateFunction("SUM");
dimColumn.setColumnName("IMEI_COUNT");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataType.STRING);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableWithComplexTypesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableWithComplexTypesTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableWithComplexTypesTest.java
index 8a5d9d7..4cf8725 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableWithComplexTypesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTableWithComplexTypesTest.java
@@ -127,7 +127,6 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
private ColumnSchema getColumnarMeasureColumn() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setAggregateFunction("SUM");
dimColumn.setColumnName("IMEI_COUNT");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataType.STRING);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
index 0ce6c1e..9ff0677 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
@@ -83,7 +83,7 @@ public class ColumnarKeyStoreDataHolderUnitTest {
byte[] keyBlockData = new byte[] { 32, 64, 32, 40, 64, 8, 8 };
List<byte[]> noDictionaryValBasedKeyBlockData = new java.util.ArrayList<>();
noDictionaryValBasedKeyBlockData.add(keyBlockData);
- new ColumnarKeyStoreDataHolder(noDictionaryValBasedKeyBlockData, columnarKeyStoreMetadata);
+ new ColumnarKeyStoreDataHolder(columnarKeyStoreMetadata);
int columnIndex = 0;
int expected_result = 8232;
int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
[8/9] incubator-carbondata git commit: clean up core
Posted by ra...@apache.org.
clean up core
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/eaadc88a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/eaadc88a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/eaadc88a
Branch: refs/heads/master
Commit: eaadc88a5525ae283dfc328026c01f09e7aba20f
Parents: 82072ee
Author: jackylk <ja...@huawei.com>
Authored: Wed Jan 4 20:56:40 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Wed Jan 4 20:56:40 2017 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/core/cache/Cache.java | 11 +-
.../AbstractColumnDictionaryInfo.java | 3 +-
.../dictionary/AbstractDictionaryCache.java | 105 +++--
.../dictionary/DictionaryBuilderException.java | 63 +++
.../dictionary/DoubleArrayTrieDictionary.java | 384 -------------------
.../dictionary/ForwardDictionaryCache.java | 18 +-
.../dictionary/ReverseDictionaryCache.java | 18 +-
.../core/carbon/CarbonDataLoadSchema.java | 25 +-
.../datastore/AbstractBlockIndexStoreCache.java | 14 +-
.../core/carbon/datastore/BlockIndexStore.java | 20 +-
.../core/carbon/datastore/DataRefNode.java | 16 +-
.../carbon/datastore/DataRefNodeFinder.java | 2 -
.../carbon/datastore/SegmentTaskIndexStore.java | 36 +-
.../carbon/datastore/block/AbstractIndex.java | 7 -
.../core/carbon/datastore/block/BlockIndex.java | 1 -
.../datastore/block/SegmentProperties.java | 5 +-
.../datastore/block/SegmentTaskIndex.java | 1 -
.../carbon/datastore/block/TableTaskInfo.java | 3 +-
.../chunk/DimensionColumnDataChunk.java | 3 -
.../reader/DimensionColumnChunkReader.java | 8 +-
.../chunk/reader/MeasureColumnChunkReader.java | 7 +-
.../reader/dimension/AbstractChunkReader.java | 1 -
...mpressedDimensionChunkFileBasedReaderV1.java | 5 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 7 +-
.../measure/AbstractMeasureChunkReader.java | 3 -
...CompressedMeasureChunkFileBasedReaderV1.java | 7 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 14 +-
.../exception/IndexBuilderException.java | 4 +-
.../impl/btree/AbstractBTreeBuilder.java | 3 +-
.../impl/btree/AbstractBTreeLeafNode.java | 13 +-
.../impl/btree/BTreeDataRefNodeFinder.java | 4 -
.../datastore/impl/btree/BTreeNonLeafNode.java | 5 -
.../impl/btree/BlockBTreeLeafNode.java | 1 -
.../impl/btree/BlockletBTreeLeafNode.java | 11 +-
.../compressor/ChunkCompressorMeta.java | 56 ---
.../blocklet/compressor/CompressionCodec.java | 31 --
.../metadata/blocklet/datachunk/DataChunk.java | 105 +----
.../metadata/blocklet/sort/SortState.java | 38 --
.../ThriftWrapperSchemaConverterImpl.java | 2 -
.../core/carbon/metadata/encoder/Encoding.java | 2 +-
.../metadata/schema/table/CarbonTable.java | 4 -
.../carbon/metadata/schema/table/TableInfo.java | 28 --
.../metadata/schema/table/TableSchema.java | 30 --
.../schema/table/column/CarbonMeasure.java | 19 -
.../schema/table/column/ColumnSchema.java | 21 -
.../core/carbon/path/CarbonTablePath.java | 1 -
.../QueryStatisticsRecorderDummy.java | 2 +-
.../core/compression/DoubleCompressor.java | 6 +-
.../core/constants/CarbonCommonConstants.java | 26 --
.../core/datastorage/store/FileHolder.java | 15 +-
.../core/datastorage/store/NodeKeyStore.java | 64 ----
.../datastorage/store/NodeMeasureDataStore.java | 6 -
.../store/columnar/ColumnarKeyStore.java | 47 ---
.../columnar/ColumnarKeyStoreDataHolder.java | 45 +--
.../store/columnar/ColumnarKeyStoreInfo.java | 262 -------------
.../columnar/ColumnarKeyStoreMetadata.java | 107 +-----
.../store/compression/WriterCompressModel.java | 3 +-
.../decimal/UnCompressByteArray.java | 2 +-
.../decimal/UnCompressMaxMinFloat.java | 2 +-
.../compression/none/UnCompressNoneDefault.java | 3 -
.../compression/none/UnCompressNoneLong.java | 6 +-
.../store/impl/DFSFileHolderImpl.java | 111 ++----
.../datastorage/store/impl/FileFactory.java | 68 ----
.../datastorage/store/impl/FileHolderImpl.java | 67 ++--
.../store/impl/MemoryMappedFileHolderImpl.java | 118 ------
...ractHeavyCompressedDoubleArrayDataStore.java | 8 +-
...yCompressedDoubleArrayDataInMemoryStore.java | 34 --
.../dictionary/generator/DictionaryWriter.java | 3 +-
.../IncrementalColumnDictionaryGenerator.java | 6 +-
.../generator/ServerDictionaryGenerator.java | 2 +-
.../generator/TableDictionaryGenerator.java | 9 +-
.../dictionary/server/DictionaryServer.java | 2 +-
.../core/keygenerator/KeyGenerator.java | 3 -
.../keygenerator/columnar/ColumnarSplitter.java | 18 -
.../MultiDimKeyVarLengthEquiSplitGenerator.java | 26 +-
...tiDimKeyVarLengthVariableSplitGenerator.java | 14 +-
.../TimeStampGranularityConstants.java | 4 +-
.../TimeStampGranularityTypeValue.java | 2 +-
.../mdkey/AbstractKeyGenerator.java | 2 +-
.../mdkey/MultiDimKeyVarLengthGenerator.java | 10 -
.../core/memory/HeapMemoryAllocator.java | 2 -
.../carbondata/core/metadata/BlockletInfo.java | 202 ----------
.../core/metadata/BlockletInfoColumnar.java | 49 ---
.../core/reader/CarbonDictionaryReaderImpl.java | 8 +-
.../core/reader/CarbonFooterReader.java | 5 +-
.../carbondata/core/reader/ThriftReader.java | 2 +-
.../CarbonDictionarySortIndexReader.java | 4 +-
.../core/service/ColumnUniqueIdService.java | 2 +-
.../core/service/DictionaryService.java | 10 +-
.../carbondata/core/unsafe/CarbonUnsafe.java | 2 +-
.../util/AbstractDataFileFooterConverter.java | 61 +--
.../carbondata/core/util/CarbonMergerUtil.java | 4 +-
.../core/util/CarbonMetadataUtil.java | 2 +-
.../carbondata/core/util/CarbonProperties.java | 16 -
.../core/util/CarbonTimeStatisticsFactory.java | 8 +-
.../apache/carbondata/core/util/CarbonUtil.java | 174 +++------
.../core/util/CarbonUtilException.java | 80 ----
.../carbondata/core/util/CompressionFinder.java | 2 +-
.../carbondata/core/util/DataTypeUtil.java | 2 +-
.../core/util/ValueCompressionUtil.java | 10 +-
.../core/writer/CarbonFooterWriter.java | 2 -
.../core/writer/CarbonIndexFileWriter.java | 1 -
.../core/writer/HierarchyValueWriterForCSV.java | 4 +-
.../CarbonDictionarySortIndexWriter.java | 7 +-
.../CarbonDictionarySortIndexWriterImpl.java | 5 +-
.../CarbonDictionarySortInfoPreparator.java | 4 +-
.../sortindex/CarbonDictionarySortModel.java | 12 +-
.../impl/DictionaryBasedResultCollector.java | 5 -
.../DictionaryBasedVectorResultCollector.java | 5 -
.../collector/impl/RawBasedResultCollector.java | 5 -
.../scan/complextypes/ArrayQueryType.java | 54 +--
.../scan/complextypes/ComplexQueryType.java | 12 +-
.../scan/complextypes/PrimitiveQueryType.java | 42 +-
.../scan/complextypes/StructQueryType.java | 60 +--
.../carbondata/scan/executor/QueryExecutor.java | 6 +-
.../exception/QueryExecutionException.java | 2 -
.../executor/impl/AbstractQueryExecutor.java | 47 +--
.../scan/executor/impl/DetailQueryExecutor.java | 3 +-
.../executor/impl/QueryExecutorProperties.java | 24 --
.../impl/VectorDetailQueryExecutor.java | 6 +-
.../scan/executor/infos/AggregatorInfo.java | 44 ---
.../scan/executor/infos/BlockExecutionInfo.java | 160 --------
.../scan/executor/infos/KeyStructureInfo.java | 19 -
.../scan/executor/infos/SortInfo.java | 125 ------
.../scan/executor/util/QueryUtil.java | 170 +-------
.../scan/expression/BinaryExpression.java | 16 -
.../carbondata/scan/expression/Expression.java | 1 -
.../conditional/ConditionalExpression.java | 6 -
.../exception/FilterIllegalMemberException.java | 4 -
.../exception/FilterUnsupportedException.java | 4 -
.../logical/BinaryLogicalExpression.java | 18 -
.../scan/filter/FilterExpressionProcessor.java | 22 +-
.../carbondata/scan/filter/FilterProcessor.java | 9 +-
.../carbondata/scan/filter/FilterUtil.java | 174 +++------
.../scan/filter/GenericQueryType.java | 22 +-
.../filter/executer/AndFilterExecuterImpl.java | 3 +-
.../ExcludeColGroupFilterExecuterImpl.java | 2 -
.../executer/ExcludeFilterExecuterImpl.java | 3 +-
.../scan/filter/executer/FilterExecuter.java | 4 +-
.../IncludeColGroupFilterExecuterImpl.java | 2 -
.../executer/IncludeFilterExecuterImpl.java | 8 +-
.../filter/executer/OrFilterExecuterImpl.java | 3 +-
.../executer/RowLevelFilterExecuterImpl.java | 86 ++---
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 7 +-
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 3 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 3 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 3 +-
.../RowLevelRangeTypeExecuterFacory.java | 9 +-
.../filter/resolver/AndFilterResolverImpl.java | 56 ---
.../resolver/ConditionalFilterResolverImpl.java | 39 +-
.../filter/resolver/FilterResolverIntf.java | 18 +-
.../resolver/LogicalFilterResolverImpl.java | 9 +-
.../resolver/RestructureFilterResolverImpl.java | 208 ----------
.../resolver/RowLevelFilterResolverImpl.java | 7 +-
.../RowLevelRangeFilterResolverImpl.java | 51 +--
.../DimColumnResolvedFilterInfo.java | 59 +--
.../MeasureColumnResolvedFilterInfo.java | 37 +-
.../visitable/ResolvedFilterInfoVisitable.java | 38 --
.../visitor/CustomTypeDictionaryVisitor.java | 13 +-
.../visitor/DictionaryColumnVisitor.java | 33 +-
.../visitor/NoDictionaryTypeVisitor.java | 7 +-
.../visitor/ResolvedFilterInfoVisitorIntf.java | 4 +-
.../carbondata/scan/model/CarbonQueryPlan.java | 83 ----
.../carbondata/scan/model/QueryColumn.java | 41 --
.../carbondata/scan/model/QueryModel.java | 184 +--------
.../carbondata/scan/model/SortOrderType.java | 57 ---
.../processor/AbstractDataBlockIterator.java | 21 +-
.../apache/carbondata/scan/result/Result.java | 70 ----
.../AbstractDetailQueryResultIterator.java | 1 -
.../iterator/DetailQueryResultIterator.java | 10 +-
.../scan/result/iterator/RawResultIterator.java | 2 +-
.../scan/result/vector/CarbonColumnVector.java | 24 +-
.../vector/MeasureDataVectorProcessor.java | 2 +-
.../scan/scanner/AbstractBlockletScanner.java | 8 +-
.../scan/scanner/BlockletScanner.java | 7 +-
.../scan/scanner/impl/FilterScanner.java | 29 +-
.../dictionary/ReverseDictionaryCacheTest.java | 5 +-
.../store/impl/DFSFileHolderImplUnitTest.java | 40 +-
.../store/impl/FileHolderImplUnitTest.java | 17 +-
.../MemoryMappedFileHolderImplUnitTest.java | 137 -------
.../datastore/SegmentTaskIndexStoreTest.java | 4 +-
.../datastore/block/SegmentPropertiesTest.java | 2 -
.../block/SegmentPropertiesTestUtil.java | 2 -
...ressedDimensionChunkFileBasedReaderTest.java | 5 +-
...mpressedMeasureChunkFileBasedReaderTest.java | 5 +-
.../carbon/metadata/CarbonMetadataTest.java | 1 -
.../ThriftWrapperSchemaConverterImplTest.java | 1 -
.../metadata/schema/table/CarbonTableTest.java | 1 -
.../table/CarbonTableWithComplexTypesTest.java | 1 -
.../ColumnarKeyStoreDataHolderUnitTest.java | 2 +-
.../carbondata/core/util/CarbonUtilTest.java | 56 +--
.../core/writer/CarbonFooterWriterTest.java | 4 +-
.../CarbonDictionarySortInfoPreparatorTest.java | 9 +-
.../impl/RawBasedResultCollectorTest.java | 1 -
.../carbondata/scan/filter/FilterUtilTest.java | 65 +---
.../scanner/impl/FilterScannerTest.java | 12 +-
.../carbondata/hadoop/CacheAccessClient.java | 12 +-
.../carbondata/hadoop/CarbonInputFormat.java | 41 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 35 +-
.../hadoop/readsupport/CarbonReadSupport.java | 8 +-
.../AbstractDictionaryDecodedReadSupport.java | 17 +-
.../hadoop/util/CarbonInputFormatUtil.java | 1 -
.../hadoop/test/util/StoreCreator.java | 4 +-
.../carbondata/spark/load/CarbonLoaderUtil.java | 14 +-
.../spark/merger/CarbonCompactionExecutor.java | 18 +-
.../spark/merger/CarbonCompactionUtil.java | 12 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 5 +-
.../execution/command/carbonTableSchema.scala | 1 -
.../readsupport/SparkRowReadSupportImpl.java | 3 +-
.../scala/org/apache/spark/sql/CarbonScan.scala | 10 -
.../spark/sql/SparkUnknownExpression.scala | 17 +-
.../readsupport/SparkRowReadSupportImpl.java | 4 +-
.../scala/org/apache/spark/sql/CarbonScan.scala | 3 -
.../processing/csvload/DataGraphExecuter.java | 128 ++-----
.../processing/csvload/GraphExecutionUtil.java | 2 +-
.../processing/datatypes/PrimitiveDataType.java | 9 +-
.../processing/mdkeygen/MDKeyGenStep.java | 5 +-
.../newflow/AbstractDataLoadProcessorStep.java | 5 +-
.../newflow/converter/RowConverter.java | 4 +-
.../impl/DictionaryFieldConverterImpl.java | 29 +-
.../converter/impl/FieldEncoderFactory.java | 4 +-
.../converter/impl/RowConverterImpl.java | 15 +-
.../impl/UnsafeParallelReadMergeSorterImpl.java | 2 +-
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 20 +-
.../merger/UnsafeIntermediateFileMerger.java | 3 +-
.../steps/DataConverterProcessorStepImpl.java | 4 +-
...ConverterProcessorWithBucketingStepImpl.java | 4 +-
.../steps/DataWriterProcessorStepImpl.java | 3 +-
.../newflow/steps/SortProcessorStepImpl.java | 3 +-
.../sortdata/IntermediateFileMerger.java | 3 +-
.../FileStoreSurrogateKeyGenForCSV.java | 55 +--
.../util/CarbonDataProcessorUtil.java | 40 +-
.../carbon/datastore/BlockIndexStoreTest.java | 12 +-
.../carbondata/test/util/StoreCreator.java | 4 +-
234 files changed, 946 insertions(+), 5250 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
index c87e7d9..1734385 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/Cache.java
@@ -19,10 +19,9 @@
package org.apache.carbondata.core.cache;
+import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.util.CarbonUtilException;
-
/**
* A semi-persistent mapping from keys to values. Cache entries are manually added using
* #get(Key), #getAll(List<Keys>) , and are stored in the cache until
@@ -41,9 +40,9 @@ public interface Cache<K, V> {
*
* @param key
* @return
- * @throws CarbonUtilException in case memory is not sufficient to load data into memory
+ * @throws IOException in case memory is not sufficient to load data into memory
*/
- V get(K key) throws CarbonUtilException;
+ V get(K key) throws IOException;
/**
* This method will return a list of values for the given list of keys.
@@ -51,9 +50,9 @@ public interface Cache<K, V> {
* Access count of Cacheable entry will be incremented
* @param keys
* @return
- * @throws CarbonUtilException in case memory is not sufficient to load data into memory
+ * @throws IOException in case memory is not sufficient to load data into memory
*/
- List<V> getAll(List<K> keys) throws CarbonUtilException;
+ List<V> getAll(List<K> keys) throws IOException;
/**
* This method will return the value for the given key. It will not check and load
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
index ab400f0..640a6b2 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
@@ -151,8 +151,7 @@ public abstract class AbstractColumnDictionaryInfo implements DictionaryInfo {
* @return
*/
@Override public DictionaryChunksWrapper getDictionaryChunks() {
- DictionaryChunksWrapper chunksWrapper = new DictionaryChunksWrapper(dictionaryChunks);
- return chunksWrapper;
+ return new DictionaryChunksWrapper(dictionaryChunks);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index 5fbef5d..b554d34 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -35,7 +35,6 @@ import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
import org.apache.carbondata.core.service.DictionaryService;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtilException;
/**
* Abstract class which implements methods common to reverse and forward dictionary cache
@@ -137,8 +136,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId
carbonTablePath.getDictionaryMetaFilePath(dictionaryColumnUniqueIdentifier
.getColumnIdentifier().getColumnId());
FileFactory.FileType fileType = FileFactory.getFileType(dictionaryFilePath);
- CarbonFile carbonFile = FileFactory.getCarbonFile(dictionaryFilePath, fileType);
- return carbonFile;
+ return FileFactory.getCarbonFile(dictionaryFilePath, fileType);
}
/**
@@ -149,65 +147,62 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId
* tableName and columnIdentifier
* @param dictionaryInfo
* @param lruCacheKey
- * @param loadSortIndex read and load sort index file in memory
- * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ * @throws IOException in case memory is not sufficient to load dictionary
+ * into memory
*/
protected void checkAndLoadDictionaryData(
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier,
DictionaryInfo dictionaryInfo, String lruCacheKey, boolean loadSortIndex)
- throws CarbonUtilException {
- try {
- // read last segment dictionary meta chunk entry to get the end offset of file
- CarbonFile carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
- boolean dictionaryMetaFileModified =
- isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),
- dictionaryInfo.getDictionaryMetaFileLength());
- // if dictionary metadata file is modified then only read the last entry from dictionary
- // meta file
- if (dictionaryMetaFileModified) {
- synchronized (dictionaryInfo) {
- carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
- dictionaryMetaFileModified =
- isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),
- dictionaryInfo.getDictionaryMetaFileLength());
- // Double Check :
- // if dictionary metadata file is modified then only read the last entry from dictionary
- // meta file
- if (dictionaryMetaFileModified) {
- CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk =
- readLastChunkFromDictionaryMetadataFile(dictionaryColumnUniqueIdentifier);
- // required size will be size total size of file - offset till file is
- // already read
- long requiredSize =
- carbonDictionaryColumnMetaChunk.getEnd_offset() - dictionaryInfo.getMemorySize();
- if (requiredSize > 0) {
- boolean columnAddedToLRUCache =
- carbonLRUCache.put(lruCacheKey, dictionaryInfo, requiredSize);
- // if column is successfully added to lru cache then only load the
- // dictionary data
- if (columnAddedToLRUCache) {
- // load dictionary data
- loadDictionaryData(dictionaryInfo, dictionaryColumnUniqueIdentifier,
- dictionaryInfo.getMemorySize(), carbonDictionaryColumnMetaChunk.getEnd_offset(),
- loadSortIndex);
- // set the end offset till where file is read
- dictionaryInfo
- .setOffsetTillFileIsRead(carbonDictionaryColumnMetaChunk.getEnd_offset());
- dictionaryInfo.setFileTimeStamp(carbonFile.getLastModifiedTime());
- dictionaryInfo.setDictionaryMetaFileLength(carbonFile.getSize());
- } else {
- throw new CarbonUtilException(
- "Cannot load dictionary into memory. Not enough memory available");
- }
+ throws IOException {
+ // read last segm
+ // ent dictionary meta chunk entry to get the end offset of file
+ CarbonFile carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
+ boolean dictionaryMetaFileModified =
+ isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),
+ dictionaryInfo.getDictionaryMetaFileLength());
+ // if dictionary metadata file is modified then only read the last entry from dictionary
+ // meta file
+ if (dictionaryMetaFileModified) {
+ synchronized (dictionaryInfo) {
+ carbonFile = getDictionaryMetaCarbonFile(dictionaryColumnUniqueIdentifier);
+ dictionaryMetaFileModified =
+ isDictionaryMetaFileModified(carbonFile, dictionaryInfo.getFileTimeStamp(),
+ dictionaryInfo.getDictionaryMetaFileLength());
+ // Double Check :
+ // if dictionary metadata file is modified then only read the last entry from dictionary
+ // meta file
+ if (dictionaryMetaFileModified) {
+ CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk =
+ readLastChunkFromDictionaryMetadataFile(dictionaryColumnUniqueIdentifier);
+ // required size will be size total size of file - offset till file is
+ // already read
+ long requiredSize =
+ carbonDictionaryColumnMetaChunk.getEnd_offset() - dictionaryInfo.getMemorySize();
+ if (requiredSize > 0) {
+ boolean columnAddedToLRUCache =
+ carbonLRUCache.put(lruCacheKey, dictionaryInfo, requiredSize);
+ // if column is successfully added to lru cache then only load the
+ // dictionary data
+ if (columnAddedToLRUCache) {
+ // load dictionary data
+ loadDictionaryData(dictionaryInfo, dictionaryColumnUniqueIdentifier,
+ dictionaryInfo.getMemorySize(), carbonDictionaryColumnMetaChunk.getEnd_offset(),
+ loadSortIndex);
+ // set the end offset till where file is read
+ dictionaryInfo
+ .setOffsetTillFileIsRead(carbonDictionaryColumnMetaChunk.getEnd_offset());
+ dictionaryInfo.setFileTimeStamp(carbonFile.getLastModifiedTime());
+ dictionaryInfo.setDictionaryMetaFileLength(carbonFile.getSize());
+ } else {
+ throw new DictionaryBuilderException(
+ "Cannot load dictionary into memory. Not enough memory available");
}
}
}
}
- // increment the column access count
- incrementDictionaryAccessCount(dictionaryInfo);
- } catch (IOException e) {
- throw new CarbonUtilException(e.getMessage());
}
+ // increment the column access count
+ incrementDictionaryAccessCount(dictionaryInfo);
}
/**
@@ -217,9 +212,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId
* @return
*/
protected String getLruCacheKey(String columnIdentifier, CacheType cacheType) {
- String lruCacheKey =
- columnIdentifier + CarbonCommonConstants.UNDERSCORE + cacheType.getCacheName();
- return lruCacheKey;
+ return columnIdentifier + CarbonCommonConstants.UNDERSCORE + cacheType.getCacheName();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryBuilderException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryBuilderException.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryBuilderException.java
new file mode 100644
index 0000000..39635a3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryBuilderException.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.io.IOException;
+
+public class DictionaryBuilderException extends IOException {
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 21312121L;
+
+ /**
+ * The Error message.
+ */
+ private String msg = "";
+
+ /**
+ * Constructor
+ *
+ * @param msg The error message for this exception.
+ */
+ public DictionaryBuilderException(String msg) {
+ super(msg);
+ this.msg = msg;
+ }
+
+ /**
+ * getLocalizedMessage
+ */
+ @Override public String getLocalizedMessage() {
+ return super.getLocalizedMessage();
+ }
+
+ /**
+ * getMessage
+ */
+ public String getMessage() {
+ return this.msg;
+ }
+
+ @Override public String toString() {
+ return "DictionaryBuilderException: " + msg;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
deleted file mode 100644
index 13d9233..0000000
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
+++ /dev/null
@@ -1,384 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.cache.dictionary;
-
-import java.io.*;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-/**
- * A dictionary based on DoubleArrayTrie data structure that maps enumerations
- * of byte[] to int IDs. With DoubleArrayTrie the memory footprint of the mapping
- * is minimize,d if compared to HashMap.
- * This DAT implementation is inspired by https://linux.thai.net/~thep/datrie/datrie.html
- */
-
-public class DoubleArrayTrieDictionary {
- public static final byte[] HEAD_MAGIC = new byte[]{
- 0x44, 0x41, 0x54, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74
- }; // "DATTrieDict"
- public static final int HEAD_LEN = HEAD_MAGIC.length;
-
- private static final int INIT_CAPA_VALUE = 256; // init len of double array
- private static final int BASE_ROOT_VALUE = 1; // root base value of trie root
- private static final int CHCK_ROOT_VALUE = -1; // root check value of trie root
- private static final int UUSD_ROOM_VALUE = -2; // unused position, only for zero
- private static final int EPTY_BACK_VALUE = 0; // value of empty position
-
- private static final int ENCODE_BASE_VALUE = 10; // encode start number
-
- private int[] base;
- private int[] check;
- private int size;
- private int capacity;
-
- private int id = ENCODE_BASE_VALUE;
-
- public DoubleArrayTrieDictionary() {
- base = new int[INIT_CAPA_VALUE];
- check = new int[INIT_CAPA_VALUE];
- capacity = INIT_CAPA_VALUE;
- base[0] = UUSD_ROOM_VALUE;
- check[0] = UUSD_ROOM_VALUE;
- base[1] = BASE_ROOT_VALUE;
- check[1] = CHCK_ROOT_VALUE;
- size = 2;
- }
-
- private void init(int capacity, int size, int[] base, int[] check) {
- int blen = base.length;
- int clen = check.length;
- if (capacity < size || size < 0 || blen != clen) {
- throw new IllegalArgumentException("Illegal init parameters");
- }
- this.base = new int[capacity];
- this.check = new int[capacity];
- this.capacity = capacity;
- System.arraycopy(base, 0, this.base, 0, blen);
- System.arraycopy(check, 0, this.check, 0, clen);
- this.size = size;
- }
-
- public void clear() {
- base = null;
- check = null;
- size = 0;
- capacity = 0;
- }
-
- private int reSize(int newCapacity) {
- if (newCapacity < capacity) {
- return capacity;
- }
- int[] newBase = new int[newCapacity];
- int[] newCheck = new int[newCapacity];
- if (capacity > 0) {
- System.arraycopy(base, 0, newBase, 0, capacity);
- System.arraycopy(check, 0, newCheck, 0, capacity);
- }
- base = newBase;
- check = newCheck;
- capacity = newCapacity;
- return capacity;
- }
-
- public int getSize() {
- return size;
- }
-
- public int getCapacity() {
- return capacity;
- }
-
- /**
- * Get encode value of key
- *
- * @param key
- * @return
- */
- public int getValue(String key) {
- String k = key + '\0';
- byte[] bKeys = k.getBytes();
- return getValue(bKeys);
- }
-
- /**
- * Get encode value of bKeys
- *
- * @param bKeys
- * @return
- */
- private int getValue(byte[] bKeys) {
- int from = 1;
- int to;
- int current;
- int len = bKeys.length;
- if (size == 0) return -1;
- for (int i = 0; i < len; i++) {
- current = bKeys[i] & 0xFF;
- to = base[from] + current;
- if (check[to] != from) return -1;
- int baseValue = base[to];
- if (baseValue <= -ENCODE_BASE_VALUE) {
- if (i == len - 1) {
- return -1 * baseValue;
- } else {
- return -1;
- }
- }
- from = to;
-
- }
- return -1;
- }
-
- /**
- * Get all children of one node
- *
- * @param pos
- * @return
- */
- private TreeSet<Integer> getChildren(int pos) {
- TreeSet<Integer> children = new TreeSet<Integer>();
- for (int i = 0; i < 0xFF; i++) {
- int cpos = base[pos] + i;
- if (cpos >= size) break;
- if (cpos < 0) {
- return null;
- }
- if (check[cpos] == pos) {
- children.add(new Integer(i));
- }
- }
- return children;
- }
-
- /**
- * @TODO: need to optimize performance
- *
- * Find multiple free position for {values}
- * the distance between free position should be as same as {values}
- *
- * @param values
- * @return
- */
- private int findFreeRoom(SortedSet<Integer> values) {
- int min = values.first();
- int max = values.last();
- for (int i = min + 1; i < capacity; i++) {
- if (i + max >= capacity) {
- reSize(capacity + values.size());
- }
- int res = 0;
- for (Integer v : values) {
- res = res | base[v - min + i];
- }
- if (res == EPTY_BACK_VALUE) return i - min;
- }
- return -1;
- }
-
- /**
- * Find one empty position for value
- *
- * @param value
- * @return
- */
- private int findAvailableHop(int value) {
- reSize(size + 1);
- int result = size - 1;
- for (int i = value + 1; i < capacity; i++) {
- if (base[i] == EPTY_BACK_VALUE) {
- result = i - value;
- break;
- }
- }
- return result;
- }
-
- /**
- * Resolve when conflict and reset current node and its children.
- *
- * @param start current conflict position
- * @param bKey current byte value which for processing
- * @return
- */
- private int conflict(int start, int bKey) {
- int from = start;
- int newKey = bKey;
- TreeSet<Integer> children = getChildren(from);
- children.add(new Integer(newKey));
- int newBasePos = findFreeRoom(children);
- children.remove(new Integer(newKey));
-
- int oldBasePos = base[start];
- base[start] = newBasePos;
-
- int oldPos, newPos;
- for (Integer child : children) {
- oldPos = oldBasePos + child;
- newPos = newBasePos + child;
- if (oldPos == from) from = newPos;
- base[newPos] = base[oldPos];
- check[newPos] = check[oldPos];
- if (newPos >= size) size = newPos + 1;
- if (base[oldPos] > 0) {
- TreeSet<Integer> cs = getChildren(oldPos);
- for (Integer c : cs) {
- check[base[oldPos] + c] = newPos;
- }
- }
- base[oldPos] = EPTY_BACK_VALUE;
- check[oldPos] = EPTY_BACK_VALUE;
- }
- return from;
- }
-
- /**
- * Insert element (byte[]) into DAT.
- * 1. if the element has been DAT then return.
- * 2. if position which is empty then insert directly.
- * 3. if conflict then resolve it.
- *
- * @param bKeys
- * @return
- */
- private boolean insert(byte[] bKeys) {
- int from = 1;
- int klen = bKeys.length;
- for (int i = 0; i < klen; i++) {
- int c = bKeys[i] & 0xFF;
- int to = base[from] + c;
- reSize((int) (to * 1.2) + 1);
- if (check[to] == from) {
- if (i == klen - 1) return true;
- from = to;
- } else if (check[to] == EPTY_BACK_VALUE) {
- check[to] = from;
- if (i == klen - 1) {
- base[to] = -id;
- id = id + 1;
- return true;
- } else {
- int next = bKeys[i + 1] & 0xFF;
- base[to] = findAvailableHop(next);
- from = to;
- }
- if (to >= size) size = to + 1;
- } else {
- int rConflict = conflict(from, c);
- int locate = base[rConflict] + c;
- if (check[locate] != EPTY_BACK_VALUE) {
- System.err.println("conflict");
- }
- check[locate] = rConflict;
- if (i == klen - 1) {
- base[locate] = -id;
- id = id + 1;
- } else {
- int nah = bKeys[i + 1] & 0xFF;
- base[locate] = findAvailableHop(nah);
- }
- if (locate >= size) size = locate + 1;
- from = locate;
- if (i == klen - 1) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Insert element (String) into DAT, the element will be transformed to
- * byte[] firstly then insert into DAT.
- *
- * @param key
- * @return
- */
- public boolean insert(String key) {
- String k = key + '\0';
- byte[] bKeys = k.getBytes();
- if (!insert(bKeys)) {
- return false;
- }
- return true;
- }
-
- /**
- * Serialize the DAT to data output stream
- *
- * @param out
- * @throws IOException
- */
- public void write(DataOutputStream out) throws IOException {
- out.write(HEAD_MAGIC);
- out.writeInt(capacity);
- out.writeInt(size);
- for (int i = 0; i < size; i++) {
- out.writeInt(base[i]);
- }
- for (int i = 0; i < size; i++) {
- out.writeInt(check[i]);
- }
- }
-
- /**
- * Deserialize the DAT from data input stream
- *
- * @param in
- * @throws IOException
- */
- public void read(DataInputStream in) throws IOException {
- byte[] header = new byte[HEAD_LEN];
- in.read(header);
- int comp = 0;
- for (int i = 0; i < HEAD_LEN; i++) {
- comp = HEAD_MAGIC[i] - header[i];
- if (comp != 0) break;
- }
- if (comp != 0) throw new IllegalArgumentException("Illegal file type");
- int capacity = in.readInt();
- int size = in.readInt();
- if (capacity < size || size < 0) throw new IllegalArgumentException("Illegal parameters");
- int[] base = new int[size];
- int[] check = new int[size];
- for (int i = 0; i < size; i++) {
- base[i] = in.readInt();
- }
- for (int i = 0; i < size; i++) {
- check[i] = in.readInt();
- }
- init(capacity, size, base, check);
- }
-
- /**
- * Dump double array value about Trie
- */
- public void dump(PrintStream out) {
- out.println("Capacity = " + capacity + ", Size = " + size);
- for (int i = 0; i < size; i++) {
- if (base[i] != EPTY_BACK_VALUE) {
- out.print(i + ":[" + base[i] + "," + check[i] + "], ");
- }
- }
- out.println();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
index ff30c73..2dadfe4 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
@@ -19,6 +19,7 @@
package org.apache.carbondata.core.cache.dictionary;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
@@ -31,7 +32,6 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.CarbonLRUCache;
-import org.apache.carbondata.core.util.CarbonUtilException;
/**
* This class implements methods to create dictionary cache which will hold
@@ -62,10 +62,10 @@ public class ForwardDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
* @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
* tableName and columnIdentifier
* @return dictionary
- * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ * @throws IOException in case memory is not sufficient to load dictionary into memory
*/
@Override public Dictionary get(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
- throws CarbonUtilException {
+ throws IOException {
return getDictionary(dictionaryColumnUniqueIdentifier);
}
@@ -76,11 +76,11 @@ public class ForwardDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
* @param dictionaryColumnUniqueIdentifiers unique identifier which contains dbName,
* tableName and columnIdentifier
* @return list of dictionary
- * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ * @throws IOException in case memory is not sufficient to load dictionary into memory
*/
@Override public List<Dictionary> getAll(
List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers)
- throws CarbonUtilException {
+ throws IOException {
boolean exceptionOccurredInDictionaryLoading = false;
String exceptionMessage = "";
List<Dictionary> forwardDictionaryObjectList =
@@ -90,7 +90,7 @@ public class ForwardDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
for (final DictionaryColumnUniqueIdentifier uniqueIdent : dictionaryColumnUniqueIdentifiers) {
taskSubmitList.add(executorService.submit(new Callable<Dictionary>() {
- @Override public Dictionary call() throws CarbonUtilException {
+ @Override public Dictionary call() throws IOException {
Dictionary dictionary = getDictionary(uniqueIdent);
return dictionary;
}
@@ -114,7 +114,7 @@ public class ForwardDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
if (exceptionOccurredInDictionaryLoading) {
clearDictionary(forwardDictionaryObjectList);
LOGGER.error(exceptionMessage);
- throw new CarbonUtilException(exceptionMessage);
+ throw new IOException(exceptionMessage);
}
return forwardDictionaryObjectList;
}
@@ -160,11 +160,11 @@ public class ForwardDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
* @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
* tableName and columnIdentifier
* @return dictionary
- * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ * @throws IOException in case memory is not sufficient to load dictionary into memory
*/
private Dictionary getDictionary(
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
- throws CarbonUtilException {
+ throws IOException {
Dictionary forwardDictionary = null;
// dictionary is only for primitive data type
assert (!dictionaryColumnUniqueIdentifier.getDataType().isComplexType());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
index fab767f..435f5a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
@@ -19,6 +19,7 @@
package org.apache.carbondata.core.cache.dictionary;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
@@ -31,7 +32,6 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.CarbonLRUCache;
-import org.apache.carbondata.core.util.CarbonUtilException;
/**
* This class implements methods to create dictionary cache which will hold
@@ -62,10 +62,10 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
* @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
* tableName and columnIdentifier
* @return dictionary
- * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ * @throws IOException in case memory is not sufficient to load dictionary into memory
*/
@Override public Dictionary get(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
- throws CarbonUtilException {
+ throws IOException {
return getDictionary(dictionaryColumnUniqueIdentifier);
}
@@ -76,11 +76,11 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
* @param dictionaryColumnUniqueIdentifiers unique identifier which contains dbName,
* tableName and columnIdentifier
* @return list of dictionary
- * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ * @throws IOException in case memory is not sufficient to load dictionary into memory
*/
@Override public List<Dictionary> getAll(
List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers)
- throws CarbonUtilException {
+ throws IOException {
boolean exceptionOccurredInDictionaryLoading = false;
String exceptionMessage = "";
List<Dictionary> reverseDictionaryObjectList =
@@ -90,7 +90,7 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
for (final DictionaryColumnUniqueIdentifier uniqueIdent : dictionaryColumnUniqueIdentifiers) {
taskSubmitList.add(executorService.submit(new Callable<Dictionary>() {
- @Override public Dictionary call() throws CarbonUtilException {
+ @Override public Dictionary call() throws IOException {
Dictionary dictionary = getDictionary(uniqueIdent);
return dictionary;
}
@@ -114,7 +114,7 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
if (exceptionOccurredInDictionaryLoading) {
clearDictionary(reverseDictionaryObjectList);
LOGGER.error(exceptionMessage);
- throw new CarbonUtilException(exceptionMessage);
+ throw new IOException(exceptionMessage);
}
return reverseDictionaryObjectList;
}
@@ -161,11 +161,11 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
* @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
* tableName and columnIdentifier
* @return dictionary
- * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ * @throws IOException in case memory is not sufficient to load dictionary into memory
*/
private Dictionary getDictionary(
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
- throws CarbonUtilException {
+ throws IOException {
Dictionary reverseDictionary = null;
// dictionary is only for primitive data type
assert(!dictionaryColumnUniqueIdentifier.getDataType().isComplexType());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
index 7cfefc9..45b2bcd 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
@@ -65,15 +65,6 @@ public class CarbonDataLoadSchema implements Serializable {
}
/**
- * set dimensionrelation list
- *
- * @param dimensionRelationList
- */
- public void setDimensionRelationList(List<DimensionRelation> dimensionRelationList) {
- this.dimensionRelationList = dimensionRelationList;
- }
-
- /**
* get carbontable
*
* @return carbonTable
@@ -98,11 +89,6 @@ public class CarbonDataLoadSchema implements Serializable {
private String tableName;
/**
- * dimensionSource csv path
- */
- private String dimensionSource;
-
- /**
* relation with fact and dimension table
*/
private Relation relation;
@@ -118,14 +104,12 @@ public class CarbonDataLoadSchema implements Serializable {
* constructor
*
* @param tableName - dimension table name
- * @param dimensionSource - source file path
* @param relation - fact foreign key with dimension primary key mapping
* @param columns - list of columns to be used from this dimension table
*/
- public DimensionRelation(String tableName, String dimensionSource, Relation relation,
+ public DimensionRelation(String tableName, Relation relation,
List<String> columns) {
this.tableName = tableName;
- this.dimensionSource = dimensionSource;
this.relation = relation;
this.columns = columns;
}
@@ -138,13 +122,6 @@ public class CarbonDataLoadSchema implements Serializable {
}
/**
- * @return dimensionSource
- */
- public String getDimensionSource() {
- return dimensionSource;
- }
-
- /**
* @return relation
*/
public Relation getRelation() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/AbstractBlockIndexStoreCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/AbstractBlockIndexStoreCache.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/AbstractBlockIndexStoreCache.java
index c700a06..d676c02 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/AbstractBlockIndexStoreCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/AbstractBlockIndexStoreCache.java
@@ -19,7 +19,8 @@
package org.apache.carbondata.core.carbon.datastore;
-import java.util.Arrays;
+import java.io.IOException;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -30,9 +31,9 @@ import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex;
import org.apache.carbondata.core.carbon.datastore.block.BlockInfo;
import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.carbon.datastore.block.TableBlockUniqueIdentifier;
+import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
/**
* This class validate and load the B-Tree in the executor lru cache
@@ -88,7 +89,7 @@ public abstract class AbstractBlockIndexStoreCache<K, V>
*/
protected void checkAndLoadTableBlocks(AbstractIndex tableBlock,
TableBlockUniqueIdentifier tableBlockUniqueIdentifier, String lruCacheKey)
- throws CarbonUtilException {
+ throws IOException {
// calculate the required size is
TableBlockInfo blockInfo = tableBlockUniqueIdentifier.getTableBlockInfo();
long requiredMetaSize = CarbonUtil
@@ -103,13 +104,12 @@ public abstract class AbstractBlockIndexStoreCache<K, V>
if (isTableBlockAddedToLruCache) {
// load table blocks data
// getting the data file meta data of the block
- DataFileFooter footer = CarbonUtil
- .readMetadatFile(blockInfo);
+ DataFileFooter footer = CarbonUtil.readMetadatFile(blockInfo);
footer.setBlockInfo(new BlockInfo(blockInfo));
// building the block
- tableBlock.buildIndex(Arrays.asList(footer));
+ tableBlock.buildIndex(Collections.singletonList(footer));
} else {
- throw new CarbonUtilException(
+ throw new IndexBuilderException(
"Cannot load table blocks into memory. Not enough memory available");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BlockIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BlockIndexStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BlockIndexStore.java
index a452338..e431805 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BlockIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BlockIndexStore.java
@@ -19,6 +19,7 @@
package org.apache.carbondata.core.carbon.datastore;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -43,7 +44,6 @@ import org.apache.carbondata.core.carbon.datastore.block.TableBlockUniqueIdentif
import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtilException;
/**
* This class is used to load the B-Tree in Executor LRU Cache
@@ -64,10 +64,9 @@ public class BlockIndexStore<K, V> extends AbstractBlockIndexStoreCache<K, V> {
*
* @param tableBlockUniqueIdentifier Uniquely identifies the block
* @return returns the blocks B-Tree meta
- * @throws CarbonUtilException
*/
@Override public AbstractIndex get(TableBlockUniqueIdentifier tableBlockUniqueIdentifier)
- throws CarbonUtilException {
+ throws IOException {
TableBlockInfo tableBlockInfo = tableBlockUniqueIdentifier.getTableBlockInfo();
BlockInfo blockInfo = new BlockInfo(tableBlockInfo);
String lruCacheKey =
@@ -158,10 +157,10 @@ public class BlockIndexStore<K, V> extends AbstractBlockIndexStoreCache<K, V> {
*
* @param tableBlocksInfos List of unique table blocks
* @return List<AbstractIndex>
- * @throws CarbonUtilException
+ * @throws IndexBuilderException
*/
@Override public List<AbstractIndex> getAll(List<TableBlockUniqueIdentifier> tableBlocksInfos)
- throws CarbonUtilException {
+ throws IndexBuilderException {
AbstractIndex[] loadedBlock = new AbstractIndex[tableBlocksInfos.size()];
int numberOfCores = 1;
try {
@@ -181,8 +180,7 @@ public class BlockIndexStore<K, V> extends AbstractBlockIndexStoreCache<K, V> {
try {
executor.awaitTermination(1, TimeUnit.HOURS);
} catch (InterruptedException e) {
- IndexBuilderException indexBuilderException = new IndexBuilderException(e);
- throw new CarbonUtilException(indexBuilderException.getMessage(), indexBuilderException);
+ throw new IndexBuilderException(e);
}
// fill the block which were not loaded before to loaded blocks array
fillLoadedBlocks(loadedBlock, blocksList);
@@ -241,10 +239,10 @@ public class BlockIndexStore<K, V> extends AbstractBlockIndexStoreCache<K, V> {
*
* @param loadedBlockArray array of blocks which will be filled
* @param blocksList blocks loaded in thread
- * @throws CarbonUtilException in case of any failure
+ * @throws IndexBuilderException in case of any failure
*/
private void fillLoadedBlocks(AbstractIndex[] loadedBlockArray,
- List<Future<AbstractIndex>> blocksList) throws CarbonUtilException {
+ List<Future<AbstractIndex>> blocksList) throws IndexBuilderException {
int blockCounter = 0;
boolean exceptionOccurred = false;
Throwable exceptionRef = null;
@@ -260,7 +258,7 @@ public class BlockIndexStore<K, V> extends AbstractBlockIndexStoreCache<K, V> {
LOGGER.error("Block B-Tree loading failed. Clearing the access count of the loaded blocks.");
// in case of any failure clear the access count for the valid loaded blocks
clearAccessCountForLoadedBlocks(loadedBlockArray);
- throw new CarbonUtilException("Block B-tree loading failed", exceptionRef);
+ throw new IndexBuilderException("Block B-tree loading failed", exceptionRef);
}
}
@@ -295,7 +293,7 @@ public class BlockIndexStore<K, V> extends AbstractBlockIndexStoreCache<K, V> {
}
private AbstractIndex loadBlock(TableBlockUniqueIdentifier tableBlockUniqueIdentifier)
- throws CarbonUtilException {
+ throws IOException {
AbstractIndex tableBlock = new BlockIndex();
BlockInfo blockInfo = new BlockInfo(tableBlockUniqueIdentifier.getTableBlockInfo());
String lruCacheKey =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNode.java
index 0ddd8c5..28131d6 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNode.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.core.carbon.datastore;
+import java.io.IOException;
+
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastorage.store.FileHolder;
@@ -55,7 +57,6 @@ public interface DataRefNode {
* This method will be used to get the max value of all the columns this can
* be used in case of filter query
*
- * @param max value of all the columns
*/
byte[][] getColumnsMaxValue();
@@ -63,7 +64,6 @@ public interface DataRefNode {
* This method will be used to get the min value of all the columns this can
* be used in case of filter query
*
- * @param min value of all the columns
*/
byte[][] getColumnsMinValue();
@@ -79,16 +79,17 @@ public interface DataRefNode {
* data in On IO
* @return dimension data chunks
*/
- DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader, int[][] blockIndexes);
+ DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader, int[][] blockIndexes)
+ throws IOException;
/**
* Below method will be used to get the dimension chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndex block index to be read
* @return dimension data chunk
*/
- DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader, int blockIndexes);
+ DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader, int blockIndexes)
+ throws IOException;
/**
* Below method will be used to get the measure chunk
@@ -102,7 +103,8 @@ public interface DataRefNode {
* data in On IO
* @return measure column data chunk
*/
- MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader, int[][] blockIndexes);
+ MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
+ throws IOException;
/**
* Below method will be used to read the measure chunk
@@ -111,5 +113,5 @@ public interface DataRefNode {
* @param blockIndex block index to be read from file
* @return measure data chunk
*/
- MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex);
+ MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNodeFinder.java
index 78592f7..c8edc0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNodeFinder.java
@@ -28,7 +28,6 @@ public interface DataRefNodeFinder {
* the search key
*
* @param dataBlocks complete data blocks present
- * @param serachKey key to be search
* @return data block
*/
DataRefNode findFirstDataBlock(DataRefNode dataBlocks, IndexKey searchKey);
@@ -38,7 +37,6 @@ public interface DataRefNodeFinder {
* the search key
*
* @param dataBlocks complete data blocks present
- * @param serachKey key to be search
* @return data block
*/
DataRefNode findLastDataBlock(DataRefNode dataBlocks, IndexKey searchKey);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
index ef0ac62..eb8aea2 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.core.carbon.datastore;
+import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
@@ -40,7 +41,6 @@ import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderExcepti
import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.carbon.path.CarbonTablePath.DataFileUtil;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
/**
* Class to handle loading, unloading,clearing,storing of the table
@@ -81,16 +81,11 @@ public class SegmentTaskIndexStore
@Override
public SegmentTaskIndexWrapper get(TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier)
- throws CarbonUtilException {
- SegmentTaskIndexWrapper segmentTaskIndexWrapper = null;
- try {
- segmentTaskIndexWrapper =
- loadAndGetTaskIdToSegmentsMap(tableSegmentUniqueIdentifier.getSegmentToTableBlocksInfos(),
- tableSegmentUniqueIdentifier.getAbsoluteTableIdentifier(),
- tableSegmentUniqueIdentifier);
- } catch (IndexBuilderException e) {
- throw new CarbonUtilException(e.getMessage(), e);
- }
+ throws IOException {
+ SegmentTaskIndexWrapper segmentTaskIndexWrapper =
+ loadAndGetTaskIdToSegmentsMap(tableSegmentUniqueIdentifier.getSegmentToTableBlocksInfos(),
+ tableSegmentUniqueIdentifier.getAbsoluteTableIdentifier(),
+ tableSegmentUniqueIdentifier);
if (null != segmentTaskIndexWrapper) {
segmentTaskIndexWrapper.incrementAccessCount();
}
@@ -102,21 +97,21 @@ public class SegmentTaskIndexStore
*
* @param tableSegmentUniqueIdentifiers
* @return
- * @throws CarbonUtilException
+ * @throws IOException
*/
@Override public List<SegmentTaskIndexWrapper> getAll(
- List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers) throws CarbonUtilException {
+ List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException {
List<SegmentTaskIndexWrapper> segmentTaskIndexWrappers =
new ArrayList<>(tableSegmentUniqueIdentifiers.size());
try {
for (TableSegmentUniqueIdentifier segmentUniqueIdentifier : tableSegmentUniqueIdentifiers) {
segmentTaskIndexWrappers.add(get(segmentUniqueIdentifier));
}
- } catch (CarbonUtilException e) {
+ } catch (IOException e) {
for (SegmentTaskIndexWrapper segmentTaskIndexWrapper : segmentTaskIndexWrappers) {
segmentTaskIndexWrapper.clear();
}
- throw new CarbonUtilException("Problem in loading segment blocks.", e);
+ throw e;
}
return segmentTaskIndexWrappers;
}
@@ -155,13 +150,12 @@ public class SegmentTaskIndexStore
* @param segmentToTableBlocksInfos segment id to block info
* @param absoluteTableIdentifier absolute table identifier
* @return map of taks id to segment mapping
- * @throws IndexBuilderException
+ * @throws IOException
*/
private SegmentTaskIndexWrapper loadAndGetTaskIdToSegmentsMap(
Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos,
AbsoluteTableIdentifier absoluteTableIdentifier,
- TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier)
- throws IndexBuilderException, CarbonUtilException {
+ TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier) throws IOException {
// task id to segment map
Iterator<Map.Entry<String, List<TableBlockInfo>>> iteratorOverSegmentBlocksInfos =
segmentToTableBlocksInfos.entrySet().iterator();
@@ -228,7 +222,7 @@ public class SegmentTaskIndexStore
}
} catch (IndexBuilderException e) {
LOGGER.error("Problem while loading the segment");
- throw new IndexBuilderException(e);
+ throw e;
}
return segmentTaskIndexWrapper;
}
@@ -304,11 +298,11 @@ public class SegmentTaskIndexStore
*
* @param tableBlockInfoList
* @return loaded segment
- * @throws CarbonUtilException
+ * @throws IOException
*/
private AbstractIndex loadBlocks(TaskBucketHolder taskBucketHolder,
List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier tableIdentifier)
- throws CarbonUtilException {
+ throws IOException {
// all the block of one task id will be loaded together
// so creating a list which will have all the data file meta data to of one task
List<DataFileFooter> footerList = CarbonUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java
index dd712fa..4dd76c6 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java
@@ -53,13 +53,6 @@ public abstract class AbstractIndex implements Cacheable {
protected long memorySize;
/**
- * @return the totalNumberOfRows
- */
- public long getTotalNumberOfRows() {
- return totalNumberOfRows;
- }
-
- /**
* @return the segmentProperties
*/
public SegmentProperties getSegmentProperties() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java
index cfdb127..75ebb84 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java
@@ -34,7 +34,6 @@ public class BlockIndex extends AbstractIndex {
/**
* Below method will be used to load the data block
*
- * @param blockInfo block detail
*/
public void buildIndex(List<DataFileFooter> footerList) {
// create a metadata details
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentProperties.java
index 05ad4e6..83d68ef 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentProperties.java
@@ -160,7 +160,7 @@ public class SegmentProperties {
new HashMap<Integer, Set<Integer>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
measuresOrdinalToBlockMapping =
new HashMap<Integer, Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- intialiseColGroups(columnsInTable);
+ intialiseColGroups();
fillOrdinalToBlockMappingForDimension();
fillOrdinalToBlockIndexMappingForMeasureColumns();
fillColumnGroupAndItsCardinality(columnCardinality);
@@ -171,9 +171,8 @@ public class SegmentProperties {
* it fills column groups
* e.g {{1},{2,3,4},{5},{6},{7,8,9}}
*
- * @param columnsInTable
*/
- private void intialiseColGroups(List<ColumnSchema> columnsInTable) {
+ private void intialiseColGroups() {
// StringBuffer columnGroups = new StringBuffer();
List<List<Integer>> colGrpList = new ArrayList<List<Integer>>();
List<Integer> group = new ArrayList<Integer>();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentTaskIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentTaskIndex.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentTaskIndex.java
index ce7a63a..655226f 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentTaskIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentTaskIndex.java
@@ -34,7 +34,6 @@ public class SegmentTaskIndex extends AbstractIndex {
/**
* Below method is store the blocks in some data structure
*
- * @param blockInfo block detail
*/
public void buildIndex(List<DataFileFooter> footerList) {
// create a metadata details
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
index 7ce3a14..da2cff8 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
@@ -21,6 +21,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
@@ -95,7 +96,7 @@ public class TableTaskInfo implements Distributable {
} else {
// for the case where all the nodes have same number of blocks then
// we need to return complete list instead of max node.
- if (previousValueOccurence != entry.getValue()) {
+ if (!Objects.equals(previousValueOccurence, entry.getValue())) {
useIndex = false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
index efa67e6..5bf200d 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
@@ -31,7 +31,6 @@ public interface DimensionColumnDataChunk<T> {
*
* @param data data to filed
* @param offset offset from which data need to be filed
- * @param rowId row id of the chunk
* @return how many bytes was copied
*/
int fillChunkData(byte[] data, int offset, int columnIndex, KeyStructureInfo restructuringInfo);
@@ -61,8 +60,6 @@ public interface DimensionColumnDataChunk<T> {
/**
* Below method to get the data based in row id
*
- * @param row id
- * row id of the data
* @return chunk
*/
byte[] getChunkData(int columnIndex);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
index 0153211..4758da1 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.core.carbon.datastore.chunk.reader;
+import java.io.IOException;
+
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastorage.store.FileHolder;
@@ -35,7 +37,8 @@ public interface DimensionColumnChunkReader {
* @param blockIndexes blocks to be read
* @return dimension column chunks
*/
- DimensionColumnDataChunk[] readDimensionChunks(FileHolder fileReader, int[][] blockIndexes);
+ DimensionColumnDataChunk[] readDimensionChunks(FileHolder fileReader, int[][] blockIndexes)
+ throws IOException;
/**
* Below method will be used to read the chunk based on block index
@@ -44,5 +47,6 @@ public interface DimensionColumnChunkReader {
* @param blockIndex block to be read
* @return dimension column chunk
*/
- DimensionColumnDataChunk readDimensionChunk(FileHolder fileReader, int blockIndex);
+ DimensionColumnDataChunk readDimensionChunk(FileHolder fileReader, int blockIndex)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
index 523a14e..9421ebf 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.core.carbon.datastore.chunk.reader;
+import java.io.IOException;
+
import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastorage.store.FileHolder;
@@ -33,7 +35,8 @@ public interface MeasureColumnChunkReader {
* @param blockIndexes blocks to be read
* @return measure data chunks
*/
- MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader, int[][] blockIndexes);
+ MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
+ throws IOException;
/**
* Method to read the blocks data based on block index
@@ -42,6 +45,6 @@ public interface MeasureColumnChunkReader {
* @param blockIndex block to be read
* @return measure data chunk
*/
- MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex);
+ MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
index cfe7079..6638c96 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
@@ -67,7 +67,6 @@ public abstract class AbstractChunkReader implements DimensionColumnChunkReader
* Constructor to get minimum parameter to create
* instance of this class
*
- * @param dimensionColumnChunk dimension chunk metadata
* @param eachColumnValueSize size of the each column value
* @param filePath file from which data will be read
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index 00ad35f..abacbd4 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension.v1;
+import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
@@ -65,7 +66,7 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
* @return dimension column chunks
*/
@Override public DimensionColumnDataChunk[] readDimensionChunks(FileHolder fileReader,
- int[][] blockIndexes) {
+ int[][] blockIndexes) throws IOException {
// read the column chunk based on block index and add
DimensionColumnDataChunk[] dataChunks =
new DimensionColumnDataChunk[dimensionColumnChunk.size()];
@@ -85,7 +86,7 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
* @return dimension column chunk
*/
@Override public DimensionColumnDataChunk readDimensionChunk(FileHolder fileReader,
- int blockIndex) {
+ int blockIndex) throws IOException {
byte[] dataPage = null;
int[] invertedIndexes = null;
int[] invertedIndexesReverse = null;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index 810fe08..7843d05 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension.v2;
+import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
@@ -77,7 +78,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
* @return dimension column chunks
*/
@Override public DimensionColumnDataChunk[] readDimensionChunks(final FileHolder fileReader,
- final int[][] blockIndexes) {
+ final int[][] blockIndexes) throws IOException {
// read the column chunk based on block index and add
DimensionColumnDataChunk[] dataChunks =
new DimensionColumnDataChunk[dimensionChunksOffset.size()];
@@ -122,7 +123,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
* @return dimension column chunk
*/
@Override public DimensionColumnDataChunk readDimensionChunk(FileHolder fileReader,
- int blockIndex) {
+ int blockIndex) throws IOException {
byte[] dataPage = null;
int[] invertedIndexes = null;
int[] invertedIndexesReverse = null;
@@ -217,7 +218,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
* @return dimension column chunk array
*/
private DimensionColumnDataChunk[] readDimensionChunksInGroup(FileHolder fileReader,
- int startBlockIndex, int endBlockIndex) {
+ int startBlockIndex, int endBlockIndex) throws IOException {
long currentDimensionOffset = dimensionChunksOffset.get(startBlockIndex);
byte[] data = fileReader.readByteArray(filePath, currentDimensionOffset,
(int) (dimensionChunksOffset.get(endBlockIndex + 1) - currentDimensionOffset));
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
index 6c74379..0a44336 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
@@ -33,9 +33,6 @@ public abstract class AbstractMeasureChunkReader implements MeasureColumnChunkRe
/**
* Constructor to get minimum parameter to create instance of this class
*
- * @param measureColumnChunk measure chunk metadata
- * @param compression model metadata which was to used to compress and uncompress
- * the measure value
* @param filePath file from which data will be read
*/
public AbstractMeasureChunkReader(String filePath) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 8bd9245..3dd0d44 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.core.carbon.datastore.chunk.reader.measure.v1;
+import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
@@ -27,8 +28,6 @@ import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.datastorage.store.FileHolder;
import org.apache.carbondata.core.datastorage.store.compression.ReaderCompressModel;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
-
-
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.util.ValueCompressionUtil;
@@ -63,7 +62,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
* @return measure data chunks
*/
@Override public MeasureColumnDataChunk[] readMeasureChunks(final FileHolder fileReader,
- final int[][] blockIndexes) {
+ final int[][] blockIndexes) throws IOException {
MeasureColumnDataChunk[] datChunk = new MeasureColumnDataChunk[measureColumnChunks.size()];
for (int i = 0; i < blockIndexes.length; i++) {
for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) {
@@ -81,7 +80,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
* @return measure data chunk
*/
@Override public MeasureColumnDataChunk readMeasureChunk(final FileHolder fileReader,
- final int blockIndex) {
+ final int blockIndex) throws IOException {
ValueEncoderMeta meta = measureColumnChunks.get(blockIndex).getValueEncoderMeta().get(0);
ReaderCompressModel compressModel = ValueCompressionUtil.getReaderCompressModel(meta);
UnCompressValue values = compressModel.getUnCompressValues().getNew().getCompressorObject();
[2/9] incubator-carbondata git commit: clean up core
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 4eed732..78d3185 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -121,15 +121,15 @@ public class CarbonUtilTest {
}
@Test public void testToDeleteFolderForValidPath()
- throws CarbonUtilException, InterruptedException {
+ throws InterruptedException, IOException {
File testDir = new File("../core/src/test/resources/testDir");
testDir.mkdirs();
CarbonUtil.deleteFoldersAndFiles(testDir);
assertTrue(!testDir.isDirectory());
}
- @Test(expected = CarbonUtilException.class) public void testToDeleteFolderWithIOException()
- throws CarbonUtilException, InterruptedException {
+ @Test(expected = IOException.class) public void testToDeleteFolderWithIOException()
+ throws InterruptedException, IOException {
File testDir = new File("../core/src/test/resources/testDir");
new MockUp<UserGroupInformation>() {
@SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
@@ -140,9 +140,9 @@ public class CarbonUtilTest {
CarbonUtil.deleteFoldersAndFiles(testDir);
}
- @Test(expected = CarbonUtilException.class)
+ @Test(expected = InterruptedException.class)
public void testToDeleteFolderWithInterruptedException()
- throws CarbonUtilException, InterruptedException {
+ throws InterruptedException, IOException {
File testDir = new File("../core/src/test/resources/testDir");
new MockUp<UserGroupInformation>() {
@SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
@@ -154,7 +154,7 @@ public class CarbonUtilTest {
}
@Test public void testToDeleteFileForValidPath()
- throws CarbonUtilException, InterruptedException {
+ throws InterruptedException, IOException {
File testDir = new File("../core/src/test/resources/testDir/testFile.csv");
testDir.mkdirs();
CarbonUtil.deleteFoldersAndFiles(testDir);
@@ -162,7 +162,7 @@ public class CarbonUtilTest {
}
@Test public void testToDeleteFoldersAndFilesForValidFolder()
- throws CarbonUtilException, InterruptedException {
+ throws InterruptedException, IOException {
String folderPath = "../core/src/test/resources/testDir/carbonDir";
new File(folderPath).mkdirs();
LocalCarbonFile testDir = new LocalCarbonFile(folderPath);
@@ -170,8 +170,8 @@ public class CarbonUtilTest {
assertTrue(!testDir.exists());
}
- @Test(expected = CarbonUtilException.class) public void testToDeleteFoldersAndFilesWithIOException()
- throws CarbonUtilException, InterruptedException {
+ @Test(expected = IOException.class) public void testToDeleteFoldersAndFilesWithIOException()
+ throws InterruptedException, IOException {
LocalCarbonFile testDir = new LocalCarbonFile("../core/src/test/resources/testDir/carbonDir");
new MockUp<UserGroupInformation>() {
@SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
@@ -182,8 +182,8 @@ public class CarbonUtilTest {
CarbonUtil.deleteFoldersAndFiles(testDir);
}
- @Test(expected = CarbonUtilException.class) public void testToDeleteFoldersAndFilesWithInterruptedException()
- throws CarbonUtilException, InterruptedException {
+ @Test(expected = InterruptedException.class) public void testToDeleteFoldersAndFilesWithInterruptedException()
+ throws InterruptedException, IOException {
LocalCarbonFile testDir = new LocalCarbonFile("../core/src/test/resources/testDir/carbonDir");
new MockUp<UserGroupInformation>() {
@SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
@@ -195,7 +195,7 @@ public class CarbonUtilTest {
}
@Test public void testToDeleteFoldersAndFilesForValidCarbonFile()
- throws CarbonUtilException, InterruptedException {
+ throws InterruptedException, IOException {
LocalCarbonFile testDir =
new LocalCarbonFile("../core/src/test/resources/testDir/testCarbonFile");
testDir.createNewFile();
@@ -203,7 +203,7 @@ public class CarbonUtilTest {
assertTrue(!testDir.exists());
}
- @Test public void testToGetBadLogPath() throws CarbonUtilException, InterruptedException {
+ @Test public void testToGetBadLogPath() throws InterruptedException {
new MockUp<CarbonProperties>() {
@SuppressWarnings("unused") @Mock public String getProperty(String key) {
return "../unibi-solutions/system/carbon/badRecords";
@@ -214,16 +214,16 @@ public class CarbonUtilTest {
}
@Test public void testToDeleteFoldersAndFilesForCarbonFileSilently()
- throws CarbonUtilException, InterruptedException {
+ throws IOException, InterruptedException {
LocalCarbonFile testDir = new LocalCarbonFile("../core/src/test/resources/testDir");
testDir.createNewFile();
CarbonUtil.deleteFoldersAndFilesSilent(testDir);
assertTrue(!testDir.exists());
}
- @Test(expected = CarbonUtilException.class)
+ @Test(expected = IOException.class)
public void testToDeleteFoldersAndFilesSintlyWithIOException()
- throws CarbonUtilException, IOException {
+ throws IOException, InterruptedException {
new MockUp<UserGroupInformation>() {
@SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
throws IOException {
@@ -235,9 +235,9 @@ public class CarbonUtilTest {
CarbonUtil.deleteFoldersAndFilesSilent(testDir);
}
- @Test(expected = CarbonUtilException.class)
+ @Test(expected = InterruptedException.class)
public void testToDeleteFoldersAndFilesSintlyWithInterruptedException()
- throws CarbonUtilException, IOException {
+ throws IOException, InterruptedException {
new MockUp<UserGroupInformation>() {
@SuppressWarnings("unused") @Mock public UserGroupInformation getLoginUser()
throws InterruptedException {
@@ -249,7 +249,7 @@ public class CarbonUtilTest {
CarbonUtil.deleteFoldersAndFilesSilent(testDir);
}
- @Test public void testToDeleteFiles() throws IOException, CarbonUtilException {
+ @Test public void testToDeleteFiles() throws IOException {
String baseDirectory = "../core/src/test/resources/";
File file1 = new File(baseDirectory + "File1.txt");
File file2 = new File(baseDirectory + "File2.txt");
@@ -341,7 +341,8 @@ public class CarbonUtilTest {
.exists());
}
- @Test public void testToGetCardinalityFromLevelMetadataFile() throws CarbonUtilException {
+ @Test public void testToGetCardinalityFromLevelMetadataFile()
+ throws IOException, InterruptedException {
int[] cardinality = CarbonUtil.getCardinalityFromLevelMetadataFile(
"../core/src/test/resources/testDatabase/levelmetadata_testTable.metadata");
int[] expectedCardinality = { 10, 20, 30, 40 };
@@ -351,7 +352,7 @@ public class CarbonUtilTest {
}
@Test public void testToGetCardinalityFromLevelMetadataFileForInvalidPath()
- throws CarbonUtilException {
+ throws IOException, InterruptedException {
int[] cardinality = CarbonUtil.getCardinalityFromLevelMetadataFile("");
assertEquals(cardinality, null);
}
@@ -550,7 +551,7 @@ public class CarbonUtilTest {
}
}
- @Test public void testToReadMetadatFile() throws CarbonUtilException {
+ @Test public void testToReadMetadatFile() throws IOException {
new MockUp<DataFileFooterConverter>() {
@SuppressWarnings("unused") @Mock
public DataFileFooter readDataFileFooter(TableBlockInfo info) {
@@ -564,7 +565,8 @@ public class CarbonUtilTest {
assertEquals(CarbonUtil.readMetadatFile(info).getVersionId().number(), 1);
}
- @Test(expected = CarbonUtilException.class) public void testToReadMetadatFileWithException()
+ @Test(expected = IOException.class)
+ public void testToReadMetadatFileWithException()
throws Exception {
TableBlockInfo info = new TableBlockInfo("file:/", 1, "0", new String[0], 1, ColumnarFormatVersion.V1);
CarbonUtil.readMetadatFile(info);
@@ -632,7 +634,8 @@ public class CarbonUtilTest {
file.deleteOnExit();
}
- @Test public void testToReadHeaderWithFileNotFoundException() throws IOException {
+ @Test(expected = IOException.class)
+ public void testToReadHeaderWithFileNotFoundException() throws IOException {
new MockUp<FileFactory>() {
@SuppressWarnings("unused") @Mock
public DataInputStream getDataInputStream(String path, FileFactory.FileType fileType)
@@ -644,7 +647,8 @@ public class CarbonUtilTest {
assertEquals(null, result);
}
- @Test public void testToReadHeaderWithIOException() throws IOException {
+ @Test(expected = IOException.class)
+ public void testToReadHeaderWithIOException() throws IOException {
new MockUp<FileFactory>() {
@SuppressWarnings("unused") @Mock
public DataInputStream getDataInputStream(String path, FileFactory.FileType fileType)
@@ -688,7 +692,7 @@ public class CarbonUtilTest {
List<Encoding> encodingList = new ArrayList<>();
encodingList.add(Encoding.DELTA);
- dataChunk.setEncoderList(encodingList);
+ dataChunk.setEncodingList(encodingList);
List<ValueEncoderMeta> valueEncoderMetas = new ArrayList<>();
ValueEncoderMeta valueEncoderMeta = new ValueEncoderMeta();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
index 20a0790..432dfbf 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
@@ -122,8 +122,8 @@ public class CarbonFooterWriterTest extends TestCase{
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
- dimColumn.setNumberOfChild(0);;
- return dimColumn;
+ dimColumn.setNumberOfChild(0);
+ return dimColumn;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
index e0c444b..246fe1d 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.CarbonUtilException;
import mockit.Mock;
import mockit.MockUp;
@@ -47,10 +46,8 @@ public class CarbonDictionarySortInfoPreparatorTest {
/**
* Tests the getDictionarySortInfo method
- *
- * @throws CarbonUtilException
*/
- @Test public void testGetDictionarySortInfo() throws CarbonUtilException {
+ @Test public void testGetDictionarySortInfo() {
List<String> newDistinctValues = new ArrayList<>();
newDistinctValues.add("abc");
@@ -83,10 +80,8 @@ public class CarbonDictionarySortInfoPreparatorTest {
/**
* Tests getDictionarySortInfo when dictionary is null
- *
- * @throws CarbonUtilException
*/
- @Test public void testGetDictionarySortInfoDictionaryNullCase() throws CarbonUtilException {
+ @Test public void testGetDictionarySortInfoDictionaryNullCase() {
List<String> newDistinctValues = new ArrayList<>();
newDistinctValues.add("abc");
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
index 78b5916..0d75942 100644
--- a/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
+++ b/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
@@ -69,7 +69,6 @@ public class RawBasedResultCollectorTest {
KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
keyStructureInfo.setKeyGenerator(keyGenerator);
keyStructureInfo.setMaxKey(new byte[] { 1, 2 });
- keyStructureInfo.setMaskedBytes(new int[] { 1, 2 });
keyStructureInfo.setMaskByteRanges(new int[] { 1, 2 });
blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
AggregatorInfo aggregatorInfo = new AggregatorInfo();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
index 53fdc09..c93713a 100644
--- a/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
@@ -38,7 +38,6 @@ import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSche
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.ColumnExpression;
import org.apache.carbondata.scan.expression.Expression;
import org.apache.carbondata.scan.expression.LiteralExpression;
@@ -334,67 +333,6 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
defaultSurrogate) instanceof DimColumnFilterInfo);
}
- @Test public void testGetFilterList() throws Exception {
- Expression expression = new ColumnExpression("IMEI", DataType.STRING);
- ColumnExpression columnExpression = new ColumnExpression("IMEI", DataType.STRING);
- int ordinal = 1;
- boolean isIncludeFilter = true;
- CarbonTableIdentifier carbonTableIdentifier =
- new CarbonTableIdentifier("databaseName", "tableName", "tableID");
- AbsoluteTableIdentifier absoluteTableIdentifier =
- new AbsoluteTableIdentifier(this.carbonStorePath, carbonTableIdentifier);
- final Map<String, String> columnProperties = new HashMap<>();
- columnProperties.put("1", "column1");
- columnProperties.put("2", "column2");
- final CarbonTableIdentifier carbonTableIdentifier1 =
- new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
- int keyOrdinal = 1;
- int columnGroupOrdinal = 1;
- int complexTypeOrdinal = 1;
- final CarbonDimension carbonDimension =
- new CarbonDimension(columnSchema, ordinal, keyOrdinal, columnGroupOrdinal,
- complexTypeOrdinal);
-
- new MockUp<AbsoluteTableIdentifier>() {
- @Mock public CarbonTableIdentifier getCarbonTableIdentifier() {
- return carbonTableIdentifier1;
- }
- };
-
- final Map<String, String> columnProperties1 = new HashMap<>();
- columnProperties1.put("1", "column1");
- columnProperties1.put("2", "column2");
-
- new MockUp<CarbonColumn>() {
- @Mock public ColumnIdentifier getColumnIdentifier() {
- ColumnIdentifier columnIdentifier =
- new ColumnIdentifier("1", columnProperties, DataType.STRING);
- return columnIdentifier;
- }
- };
-
- new MockUp<CarbonColumn>() {
- @Mock public DataType getDataType() {
- return DataType.STRING;
- }
- };
-
- new MockUp<ColumnExpression>() {
- @Mock public CarbonDimension getDimension() {
- return carbonDimension;
- }
- };
-
- final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal, -1);
- new MockUp<ColumnExpression>() {
- @Mock public CarbonColumn getCarbonColumn() {
- return carbonColumn;
- }
- };
- assertTrue(FilterUtil.getFilterList(absoluteTableIdentifier, expression, columnExpression,
- isIncludeFilter) instanceof DimColumnFilterInfo);
- }
-
@Test public void testCheckIfDataTypeNotTimeStamp() {
Expression expression = new ColumnExpression("test", DataType.STRING);
boolean result = FilterUtil.checkIfDataTypeNotTimeStamp(expression);
@@ -437,8 +375,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
List<String> evaluateResultListFinal = new ArrayList<>();
evaluateResultListFinal.add("test1");
evaluateResultListFinal.add("test2");
- assertTrue(FilterUtil.getNoDictionaryValKeyMemberForFilter(absoluteTableIdentifier, expression,
- evaluateResultListFinal, isIncludeFilter) instanceof DimColumnFilterInfo);
+ assertTrue(FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter) instanceof DimColumnFilterInfo);
}
@Test public void testPrepareDefaultStartIndexKey() throws KeyGenException {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java b/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
index 3ff131b..d1396e5 100644
--- a/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
+++ b/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scanner.impl;
+import java.io.IOException;
import java.util.Arrays;
import java.util.BitSet;
import java.util.HashMap;
@@ -41,7 +42,6 @@ import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorder
import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorderImpl;
import org.apache.carbondata.core.datastorage.store.FileHolder;
import org.apache.carbondata.core.datastorage.store.impl.DFSFileHolderImpl;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
@@ -103,7 +103,8 @@ public class FilterScannerTest {
bTreeBuilderInfo = new BTreeBuilderInfo(Arrays.asList(dataFileFooter), new int[] { 1 });
}
- @Test public void testToScanBlockletWithEmptyBitSet() throws QueryExecutionException {
+ @Test public void testToScanBlockletWithEmptyBitSet()
+ throws IOException, FilterUnsupportedException {
new MockUp<AndFilterExecuterImpl>() {
@SuppressWarnings("unused") @Mock
public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
@@ -117,7 +118,8 @@ public class FilterScannerTest {
assertEquals(0, abstractScannedResult.numberOfOutputRows());
}
- @Test public void testToScanBlockletWithNonEmptyBitSet() throws QueryExecutionException {
+ @Test public void testToScanBlockletWithNonEmptyBitSet()
+ throws IOException, FilterUnsupportedException {
new MockUp<AndFilterExecuterImpl>() {
@SuppressWarnings("unused") @Mock
public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
@@ -173,8 +175,8 @@ public class FilterScannerTest {
assertEquals(2, abstractScannedResult.numberOfOutputRows());
}
- @Test(expected = QueryExecutionException.class) public void testToScanBlockletWithException()
- throws QueryExecutionException {
+ @Test(expected = FilterUnsupportedException.class) public void testToScanBlockletWithException()
+ throws IOException, FilterUnsupportedException {
new MockUp<AndFilterExecuterImpl>() {
@SuppressWarnings("unused") @Mock
public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheAccessClient.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheAccessClient.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheAccessClient.java
index 2760035..984930d 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheAccessClient.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheAccessClient.java
@@ -18,13 +18,12 @@
*/
package org.apache.carbondata.hadoop;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonUtilException;
/**
* CacheClient : Class used to request the segments cache
@@ -35,11 +34,6 @@ public class CacheAccessClient<K, V> {
*/
private List<K> segmentList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- /**
- * absolute table identifier
- */
- private AbsoluteTableIdentifier absoluteTableIdentifier;
-
private Cache<K, V> cache;
public CacheAccessClient(Cache cache) {
@@ -67,9 +61,9 @@ public class CacheAccessClient<K, V> {
*
* @param key
* @return
- * @throws CarbonUtilException in case memory is not sufficient to load data into memory
+ * @throws IOException in case memory is not sufficient to load data into memory
*/
- public V get(K key) throws CarbonUtilException {
+ public V get(K key) throws IOException {
V value = cache.get(key);
if (value != null) {
segmentList.add(key);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 7e9dc7a..1e4f821 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -38,7 +38,6 @@ import org.apache.carbondata.core.carbon.datastore.block.BlockletInfos;
import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.apache.carbondata.core.carbon.datastore.block.SegmentTaskIndexWrapper;
import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.carbon.datastore.impl.btree.BTreeDataRefNodeFinder;
import org.apache.carbondata.core.carbon.datastore.impl.btree.BlockBTreeLeafNode;
import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
@@ -51,14 +50,12 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
import org.apache.carbondata.hadoop.readsupport.impl.DictionaryDecodedReadSupportImpl;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
import org.apache.carbondata.hadoop.util.SchemaReader;
import org.apache.carbondata.lcm.status.SegmentStatusManager;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.Expression;
import org.apache.carbondata.scan.filter.FilterExpressionProcessor;
import org.apache.carbondata.scan.filter.FilterUtil;
@@ -240,17 +237,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
CarbonTable carbonTable = getCarbonTable(job.getConfiguration());
CarbonInputFormatUtil.processFilterExpression(filter, carbonTable);
FilterResolverIntf filterInterface = CarbonInputFormatUtil.resolveFilter(filter, identifier);
- List<InputSplit> splits;
- try {
- // do block filtering and get split
- splits = getSplits(job, filterInterface, cacheClient);
- } catch (IndexBuilderException | CarbonUtilException e) {
- throw new IOException(e);
- }
- finally {
- cacheClient.close();
- cacheClient = null;
- }
+ // do block filtering and get split
+ List<InputSplit> splits = getSplits(job, filterInterface, cacheClient);
+ cacheClient.close();
// pass the invalid segment to task side in order to remove index entry in task side
if (invalidSegments.size() > 0) {
for (InputSplit split : splits) {
@@ -286,7 +275,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
* @throws IOException
*/
private List<InputSplit> getSplits(JobContext job, FilterResolverIntf filterResolver,
- CacheClient cacheClient) throws IOException, IndexBuilderException, CarbonUtilException {
+ CacheClient cacheClient) throws IOException {
List<InputSplit> result = new LinkedList<InputSplit>();
@@ -330,8 +319,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
private List<DataRefNode> getDataBlocksOfSegment(JobContext job,
FilterExpressionProcessor filterExpressionProcessor,
AbsoluteTableIdentifier absoluteTableIdentifier, FilterResolverIntf resolver,
- String segmentId, CacheClient cacheClient)
- throws IndexBuilderException, IOException, CarbonUtilException {
+ String segmentId, CacheClient cacheClient) throws IOException {
QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
QueryStatistic statistic = new QueryStatistic();
Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap =
@@ -341,20 +329,18 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
// build result
for (AbstractIndex abstractIndex : segmentIndexMap.values()) {
-
- List<DataRefNode> filterredBlocks = null;
+ List<DataRefNode> filterredBlocks;
// if no filter is given get all blocks from Btree Index
if (null == resolver) {
filterredBlocks = getDataBlocksOfIndex(abstractIndex);
} else {
// apply filter and get matching blocks
- try {
- filterredBlocks = filterExpressionProcessor
- .getFilterredBlocks(abstractIndex.getDataRefNode(), resolver, abstractIndex,
- absoluteTableIdentifier);
- } catch (QueryExecutionException e) {
- throw new IndexBuilderException(e.getMessage());
- }
+ filterredBlocks = filterExpressionProcessor.getFilterredBlocks(
+ abstractIndex.getDataRefNode(),
+ resolver,
+ abstractIndex,
+ absoluteTableIdentifier
+ );
}
resultFilterredBlocks.addAll(filterredBlocks);
}
@@ -401,11 +387,10 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
* @param segmentId
* @return
* @throws IOException
- * @throws IndexBuilderException
*/
private Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> getSegmentAbstractIndexs(
JobContext job, AbsoluteTableIdentifier absoluteTableIdentifier, String segmentId,
- CacheClient cacheClient) throws IOException, IndexBuilderException, CarbonUtilException {
+ CacheClient cacheClient) throws IOException {
Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = null;
TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier =
new TableSegmentUniqueIdentifier(absoluteTableIdentifier, segmentId);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index 9d8b136..16f0fbb 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -37,7 +37,6 @@ import org.apache.carbondata.core.carbon.datastore.block.BlockletInfos;
import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.apache.carbondata.core.carbon.datastore.block.SegmentTaskIndexWrapper;
import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.carbon.datastore.impl.btree.BTreeDataRefNodeFinder;
import org.apache.carbondata.core.carbon.datastore.impl.btree.BlockBTreeLeafNode;
import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic;
@@ -45,13 +44,11 @@ import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsConstant
import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorder;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.hadoop.CacheClient;
import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.internal.index.Block;
import org.apache.carbondata.hadoop.internal.index.Index;
import org.apache.carbondata.hadoop.internal.segment.Segment;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.filter.FilterExpressionProcessor;
import org.apache.carbondata.scan.filter.FilterUtil;
import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
@@ -87,12 +84,8 @@ class InMemoryBTreeIndex implements Index {
AbsoluteTableIdentifier identifier = null;
//for this segment fetch blocks matching filter in BTree
- List<DataRefNode> dataRefNodes = null;
- try {
- dataRefNodes = getDataBlocksOfSegment(job, filterExpressionProcessor, identifier, filter);
- } catch (IndexBuilderException e) {
- throw new IOException(e.getMessage());
- }
+ List<DataRefNode> dataRefNodes =
+ getDataBlocksOfSegment(job, filterExpressionProcessor, identifier, filter);
for (DataRefNode dataRefNode : dataRefNodes) {
BlockBTreeLeafNode leafNode = (BlockBTreeLeafNode) dataRefNode;
TableBlockInfo tableBlockInfo = leafNode.getTableBlockInfo();
@@ -105,8 +98,7 @@ class InMemoryBTreeIndex implements Index {
}
private Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> getSegmentAbstractIndexs(
- JobContext job, AbsoluteTableIdentifier identifier)
- throws IOException, IndexBuilderException {
+ JobContext job, AbsoluteTableIdentifier identifier) throws IOException {
Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = null;
CacheClient cacheClient = new CacheClient(identifier.getStorePath());
TableSegmentUniqueIdentifier segmentUniqueIdentifier =
@@ -128,11 +120,7 @@ class InMemoryBTreeIndex implements Index {
segmentTaskIndexWrapper = cacheClient.getSegmentAccessClient().get(segmentUniqueIdentifier);
segmentIndexMap = segmentTaskIndexWrapper.getTaskIdToTableSegmentMap();
}
- }
- catch (CarbonUtilException e) {
- throw new IndexBuilderException(e.getMessage(), e);
- }
- finally {
+ } finally {
cacheClient.close();
}
return segmentIndexMap;
@@ -166,7 +154,7 @@ class InMemoryBTreeIndex implements Index {
*/
private List<DataRefNode> getDataBlocksOfSegment(JobContext job,
FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier identifier,
- FilterResolverIntf resolver) throws IndexBuilderException, IOException {
+ FilterResolverIntf resolver) throws IOException {
QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
QueryStatistic statistic = new QueryStatistic();
@@ -184,13 +172,12 @@ class InMemoryBTreeIndex implements Index {
filterredBlocks = getDataBlocksOfIndex(abstractIndex);
} else {
// apply filter and get matching blocks
- try {
- filterredBlocks =
- filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(),
- resolver, abstractIndex, identifier);
- } catch (QueryExecutionException e) {
- throw new IndexBuilderException(e.getMessage());
- }
+ filterredBlocks = filterExpressionProcessor.getFilterredBlocks(
+ abstractIndex.getDataRefNode(),
+ resolver,
+ abstractIndex,
+ identifier
+ );
}
resultFilterredBlocks.addAll(filterredBlocks);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
index 1b7f577..87da719 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.hadoop.readsupport;
+import java.io.IOException;
+
import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonColumn;
@@ -31,10 +33,10 @@ public interface CarbonReadSupport<T> {
*
* @param carbonColumns
*/
- public void initialize(CarbonColumn[] carbonColumns,
- AbsoluteTableIdentifier absoluteTableIdentifier);
+ void initialize(CarbonColumn[] carbonColumns,
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException;
- public T readRow(Object[] data);
+ T readRow(Object[] data);
/**
* This method will be used to clear the dictionary cache and update access count for each
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
index 5eda4d8..3626163 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.hadoop.readsupport.impl;
+import java.io.IOException;
+
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
@@ -28,7 +30,6 @@ import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
/**
@@ -52,7 +53,7 @@ public abstract class AbstractDictionaryDecodedReadSupport<T> implements CarbonR
* @param absoluteTableIdentifier
*/
@Override public void initialize(CarbonColumn[] carbonColumns,
- AbsoluteTableIdentifier absoluteTableIdentifier) {
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
this.carbonColumns = carbonColumns;
dictionaries = new Dictionary[carbonColumns.length];
dataTypes = new DataType[carbonColumns.length];
@@ -62,14 +63,10 @@ public abstract class AbstractDictionaryDecodedReadSupport<T> implements CarbonR
CacheProvider cacheProvider = CacheProvider.getInstance();
Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider
.createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath());
- try {
- dataTypes[i] = carbonColumns[i].getDataType();
- dictionaries[i] = forwardDictionaryCache.get(new DictionaryColumnUniqueIdentifier(
- absoluteTableIdentifier.getCarbonTableIdentifier(),
- carbonColumns[i].getColumnIdentifier(), dataTypes[i]));
- } catch (CarbonUtilException e) {
- throw new RuntimeException(e);
- }
+ dataTypes[i] = carbonColumns[i].getDataType();
+ dictionaries[i] = forwardDictionaryCache.get(new DictionaryColumnUniqueIdentifier(
+ absoluteTableIdentifier.getCarbonTableIdentifier(),
+ carbonColumns[i].getColumnIdentifier(), dataTypes[i]));
} else {
dataTypes[i] = carbonColumns[i].getDataType();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 2a70cca..9100268 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -72,7 +72,6 @@ public class CarbonInputFormatUtil {
}
}
- plan.setLimit(-1);
plan.setQueryId(System.nanoTime() + "");
return plan;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 4519e90..e983f86 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -70,7 +70,6 @@ import org.apache.carbondata.processing.dataprocessor.DataProcessTaskStatus;
import org.apache.carbondata.processing.dataprocessor.IDataProcessStatus;
import org.apache.carbondata.processing.graphgenerator.GraphGenerator;
import org.apache.carbondata.processing.graphgenerator.GraphGeneratorException;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
import java.io.*;
import java.nio.charset.Charset;
@@ -374,8 +373,7 @@ public class StoreCreator {
DataGraphExecuter graphExecuter = new DataGraphExecuter(dataProcessTaskStatus);
graphExecuter
- .executeGraph(graphPath, new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN),
- info, "0", loadModel.getSchema());
+ .executeGraph(graphPath, info, loadModel.getSchema());
// LoadMetadataDetails[] loadDetails =
// CarbonUtil.readLoadMetadata(loadModel.schema.getCarbonTable().getMetaDataFilepath());
writeLoadMetadata(loadModel.schema, loadModel.getTableName(), loadModel.getTableName(),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 419e833..7a88b63 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -63,7 +63,6 @@ import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.lcm.fileoperations.AtomicFileOperations;
import org.apache.carbondata.lcm.fileoperations.AtomicFileOperationsImpl;
import org.apache.carbondata.lcm.fileoperations.FileWriteOperation;
@@ -174,8 +173,7 @@ public final class CarbonLoaderUtil {
DataGraphExecuter graphExecuter = new DataGraphExecuter(dataProcessTaskStatus);
graphExecuter
- .executeGraph(graphPath, new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN),
- info, loadModel.getPartitionId(), loadModel.getCarbonDataLoadSchema());
+ .executeGraph(graphPath, info, loadModel.getCarbonDataLoadSchema());
}
public static List<String> addNewSliceNameToList(String newSlice, List<String> activeSlices) {
@@ -248,9 +246,7 @@ public final class CarbonLoaderUtil {
CarbonFile carbonFile = FileFactory.getCarbonFile(path, fileType);
CarbonUtil.deleteFoldersAndFiles(carbonFile);
}
- } catch (IOException e) {
- LOGGER.error("Unable to delete the given path :: " + e.getMessage());
- } catch (CarbonUtilException e) {
+ } catch (IOException | InterruptedException e) {
LOGGER.error("Unable to delete the given path :: " + e.getMessage());
}
}
@@ -313,7 +309,7 @@ public final class CarbonLoaderUtil {
try {
CarbonUtil.deleteFoldersAndFiles(new File(localStoreLocation).getParentFile());
LOGGER.info("Deleted the local store location" + localStoreLocation);
- } catch (CarbonUtilException e) {
+ } catch (IOException | InterruptedException e) {
LOGGER.error(e, "Failed to delete local data load folder location");
}
@@ -458,7 +454,7 @@ public final class CarbonLoaderUtil {
}
public static Dictionary getDictionary(DictionaryColumnUniqueIdentifier columnIdentifier,
- String carbonStorePath) throws CarbonUtilException {
+ String carbonStorePath) throws IOException {
Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache =
CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, carbonStorePath);
return dictCache.get(columnIdentifier);
@@ -466,7 +462,7 @@ public final class CarbonLoaderUtil {
public static Dictionary getDictionary(CarbonTableIdentifier tableIdentifier,
ColumnIdentifier columnIdentifier, String carbonStorePath, DataType dataType)
- throws CarbonUtilException {
+ throws IOException {
return getDictionary(
new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, dataType),
carbonStorePath);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
index ef3d2e4..e06a223 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.spark.merger;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -81,7 +82,7 @@ public class CarbonCompactionExecutor {
*
* @return List of Carbon iterators
*/
- public List<RawResultIterator> processTableBlocks() throws QueryExecutionException {
+ public List<RawResultIterator> processTableBlocks() throws QueryExecutionException, IOException {
List<RawResultIterator> resultList =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
@@ -124,7 +125,7 @@ public class CarbonCompactionExecutor {
* @return
*/
private CarbonIterator<BatchResult> executeBlockList(List<TableBlockInfo> blockList)
- throws QueryExecutionException {
+ throws QueryExecutionException, IOException {
queryModel.setTableBlockInfos(blockList);
this.queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
return queryExecutor.execute(queryModel);
@@ -165,12 +166,8 @@ public class CarbonCompactionExecutor {
* @return
*/
private QueryModel prepareQueryModel(List<TableBlockInfo> blockList) {
-
QueryModel model = new QueryModel();
-
model.setTableBlockInfos(blockList);
- model.setCountStarQuery(false);
- model.setDetailQuery(true);
model.setForcedDetailRawQuery(true);
model.setFilterExpressionResolverTree(null);
@@ -188,18 +185,9 @@ public class CarbonCompactionExecutor {
msrs.add(queryMeasure);
}
model.setQueryMeasures(msrs);
-
model.setQueryId(System.nanoTime() + "");
-
model.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
-
- model.setAggTable(false);
- model.setLimit(-1);
-
model.setTable(carbonTable);
-
- model.setInMemoryRecordSize(CarbonCommonConstants.COMPACTION_INMEMORY_RECORD_SIZE);
-
return model;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
index 404dd1a..21bd605 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
@@ -29,14 +29,12 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.carbon.datastore.block.TaskBlockInfo;
-import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.carbon.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
/**
* Utility Class for the Compaction Flow.
@@ -107,7 +105,7 @@ public class CarbonCompactionUtil {
* @return
*/
public static Map<String, List<DataFileFooter>> createDataFileFooterMappingForSegments(
- List<TableBlockInfo> tableBlockInfoList) throws IndexBuilderException {
+ List<TableBlockInfo> tableBlockInfoList) throws IOException {
Map<String, List<DataFileFooter>> segmentBlockInfoMapping = new HashMap<>();
for (TableBlockInfo blockInfo : tableBlockInfoList) {
@@ -116,18 +114,12 @@ public class CarbonCompactionUtil {
DataFileFooter dataFileMatadata = null;
// check if segId is already present in map
List<DataFileFooter> metadataList = segmentBlockInfoMapping.get(segId);
- try {
- dataFileMatadata = CarbonUtil
- .readMetadatFile(blockInfo);
- } catch (CarbonUtilException e) {
- throw new IndexBuilderException(e);
- }
+ dataFileMatadata = CarbonUtil.readMetadatFile(blockInfo);
if (null == metadataList) {
// if it is not present
eachSegmentBlocks.add(dataFileMatadata);
segmentBlockInfoMapping.put(segId, eachSegmentBlocks);
} else {
-
// if its already present then update the list.
metadataList.add(dataFileMatadata);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 94fa601..01edf4a 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -17,6 +17,7 @@
package org.apache.carbondata.spark.rdd
+import java.io.IOException
import java.util
import java.util.{Collections, List}
@@ -37,7 +38,7 @@ import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableId
import org.apache.carbondata.core.carbon.datastore.block.{Distributable, SegmentProperties, TaskBlockInfo}
import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, CarbonUtilException}
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.processing.model.CarbonLoadModel
@@ -248,7 +249,7 @@ class CarbonMergerRDD[K, V](
dataFileFooter = CarbonUtil.readMetadatFile(
CarbonInputSplit.getTableBlockInfo(carbonInputSplit))
} catch {
- case e: CarbonUtilException =>
+ case e: IOException =>
logError("Exception in preparing the data file footer for compaction " + e.getMessage)
throw e
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index ec064ed..208f993 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -163,7 +163,6 @@ class TableNewProcessor(cm: TableModel) {
val colProps = cm.colProps.get.get(colName)
colProps.asScala.foreach { x => colPropMap.put(x.key, x.value) }
}
- columnSchema.setColumnProperties(colPropMap)
columnSchema.setEncodingList(encoders)
val colUniqueIdGenerator = CarbonCommonFactory.getColumnUniqueIdGenerator
val columnUniqueId = colUniqueIdGenerator.generateUniqueId(cm.databaseName,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index 46e5244..27e6192 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.spark.readsupport;
+import java.io.IOException;
import java.sql.Date;
import java.sql.Timestamp;
@@ -35,7 +36,7 @@ import org.apache.spark.unsafe.types.UTF8String;
public class SparkRowReadSupportImpl extends AbstractDictionaryDecodedReadSupport<Row> {
@Override public void initialize(CarbonColumn[] carbonColumns,
- AbsoluteTableIdentifier absoluteTableIdentifier) {
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
super.initialize(carbonColumns, absoluteTableIdentifier);
//can initialize and generate schema here.
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index c351c30..405d9d1 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -17,9 +17,6 @@
package org.apache.spark.sql
-import java.util
-import java.util.ArrayList
-
import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
@@ -29,8 +26,6 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.execution.LeafNode
import org.apache.spark.sql.hive.CarbonMetastore
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.hadoop.CarbonProjection
import org.apache.carbondata.scan.model._
import org.apache.carbondata.spark.CarbonFilters
@@ -51,11 +46,6 @@ case class CarbonScan(
val buildCarbonPlan: CarbonQueryPlan = {
val plan: CarbonQueryPlan = new CarbonQueryPlan(relationRaw.databaseName, relationRaw.tableName)
-
- plan.setSortedDimemsions(new util.ArrayList[QueryDimension])
-
- plan.setOutLocationPath(
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS))
plan.setQueryId(ocRaw.getConf("queryId", System.nanoTime() + ""))
processFilterExpressions(plan)
plan
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
index dd51184..2c45524 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
@@ -17,15 +17,11 @@
package org.apache.spark.sql
-import java.util
-
import scala.collection.JavaConverters._
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression}
-import org.apache.spark.sql.catalyst.expressions.GenericMutableRow
+import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression, GenericMutableRow}
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding
import org.apache.carbondata.scan.expression.{ColumnExpression, ExpressionResult, UnknownExpression}
import org.apache.carbondata.scan.expression.conditional.ConditionalExpression
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException
@@ -121,7 +117,7 @@ class SparkUnknownExpression(var sparkExp: SparkExpression)
def getAllColumnListFromExpressionTree(sparkCurrentExp: SparkExpression,
- list: util.List[ColumnExpression]): util.List[ColumnExpression] = {
+ list: java.util.List[ColumnExpression]): java.util.List[ColumnExpression] = {
sparkCurrentExp match {
case carbonBoundRef: CarbonBoundReference => list.add(carbonBoundRef.colExp)
case _ => sparkCurrentExp.children.foreach(getColumnListFromExpressionTree(_, list))
@@ -129,13 +125,4 @@ class SparkUnknownExpression(var sparkExp: SparkExpression)
list
}
- def isDirectDictionaryColumns: Boolean = {
- val lst = new util.ArrayList[ColumnExpression]()
- getAllColumnListFromExpressionTree(sparkExp, lst)
- if (lst.get(0).getCarbonColumn.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- true
- } else {
- false
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index 3357d6c..a87a26c 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.spark.readsupport;
+import java.io.IOException;
+
import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonColumn;
@@ -29,7 +31,7 @@ import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
public class SparkRowReadSupportImpl extends AbstractDictionaryDecodedReadSupport<InternalRow> {
@Override public void initialize(CarbonColumn[] carbonColumns,
- AbsoluteTableIdentifier absoluteTableIdentifier) {
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
super.initialize(carbonColumns, absoluteTableIdentifier);
//can initialize and generate schema here.
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index 19c3c9c..895718d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -42,9 +42,6 @@ case class CarbonScan(
val buildCarbonPlan: CarbonQueryPlan = {
val plan: CarbonQueryPlan = new CarbonQueryPlan(relationRaw.databaseName, relationRaw.tableName)
- plan.setSortedDimemsions(new java.util.ArrayList[QueryDimension])
- plan.setOutLocationPath(
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS))
processFilterExpressions(plan)
plan
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
index 3d35837..f3fdc8e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
@@ -43,7 +43,6 @@ import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.exception.KettleException;
-import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.logging.LogLevel;
import org.pentaho.di.core.logging.LoggingObjectInterface;
import org.pentaho.di.core.logging.LoggingRegistry;
@@ -83,44 +82,25 @@ public class DataGraphExecuter {
* @return true if same, false otherwise.
*/
private boolean checkCSVAndRequestedTableColumns(String[] columnNames, String csvFilePath,
- String delimiter) {
+ String delimiter) throws IOException {
return GraphExecutionUtil.checkCSVAndRequestedTableColumns(csvFilePath, columnNames, delimiter);
}
/**
* This method returns the Columns names from the schema.
*
- * @param schemaInfo
* @param tableName
* @return column names array.
*/
- private String[] getColumnNames(SchemaInfo schemaInfo, String tableName, String partitionId,
- CarbonDataLoadSchema schema) {
-
+ private String[] getColumnNames(String tableName, CarbonDataLoadSchema schema) {
Set<String> columnNames = CarbonDataProcessorUtil.getSchemaColumnNames(schema, tableName);
-
- return columnNames.toArray(new String[columnNames.size()]);
- }
-
- /**
- * This method returns the Columns names for the dimension only from the schema.
- *
- * @param schema
- * @param schemaInfo
- * @return column names array.
- */
- private String[] getDimColumnNames(SchemaInfo schemaInfo, String factTableName,
- String dimTableName, String partitionId, CarbonDataLoadSchema schema) {
-
- Set<String> columnNames = GraphExecutionUtil.getDimensionColumnNames(dimTableName, schema);
return columnNames.toArray(new String[columnNames.size()]);
}
- private void validateCSV(SchemaInfo schemaInfo, String tableName, CarbonFile f,
- String partitionId, CarbonDataLoadSchema schema, String delimiter)
- throws DataLoadingException {
+ private void validateCSV(String tableName, CarbonFile f, CarbonDataLoadSchema schema,
+ String delimiter) throws DataLoadingException, IOException {
- String[] columnNames = getColumnNames(schemaInfo, tableName, partitionId, schema);
+ String[] columnNames = getColumnNames(tableName, schema);
if (!checkCSVAndRequestedTableColumns(columnNames, f.getAbsolutePath(), delimiter)) {
LOGGER.error(
@@ -134,14 +114,14 @@ public class DataGraphExecuter {
}
}
- public void executeGraph(String graphFilePath, List<String> measureColumns, SchemaInfo schemaInfo,
- String partitionId, CarbonDataLoadSchema schema) throws DataLoadingException {
+ public void executeGraph(String graphFilePath, SchemaInfo schemaInfo, CarbonDataLoadSchema schema)
+ throws DataLoadingException {
//This Method will validate the both fact and dimension csv files.
if (!schemaInfo.isAutoAggregateRequest() && model.getRddIteratorKey() == null ) {
- validateCSVFiles(schemaInfo, partitionId, schema);
+ validateCSVFiles(schema);
}
- execute(graphFilePath, measureColumns, schemaInfo);
+ execute(graphFilePath, schemaInfo);
}
/**
@@ -150,7 +130,7 @@ public class DataGraphExecuter {
* @throws DataLoadingException
*/
- private void execute(String graphFilePath, List<String> measureColumns, SchemaInfo schemaInfo)
+ private void execute(String graphFilePath, SchemaInfo schemaInfo)
throws DataLoadingException {
//This Method will validate the both fact and dimension csv files.
@@ -178,28 +158,18 @@ public class DataGraphExecuter {
List<StepMeta> stepsMeta = trans.getTransMeta().getSteps();
StringBuilder builder = new StringBuilder();
StringBuilder measuresInCSVFile = new StringBuilder();
- processCsvInputMeta(measureColumns, stepsMeta, builder, measuresInCSVFile);
+ processCsvInputMeta(stepsMeta, builder, measuresInCSVFile);
processGetFileNamesMeta(stepsMeta);
- processHadoopFileInputMeta(measureColumns, stepsMeta, builder, measuresInCSVFile);
+ processHadoopFileInputMeta(stepsMeta, builder, measuresInCSVFile);
}
setGraphLogLevel();
trans.execute(null);
LOGGER.info("Graph execution is started " + graphFilePath);
trans.waitUntilFinished();
LOGGER.info("Graph execution is finished.");
- } catch (KettleXMLException e) {
- LOGGER.error(e,
- "Unable to start execution of graph " + e.getMessage());
- throw new DataLoadingException("Unable to start execution of graph ", e);
-
- } catch (KettleException e) {
- LOGGER.error(e,
- "Unable to start execution of graph " + e.getMessage());
- throw new DataLoadingException("Unable to start execution of graph ", e);
- } catch (Throwable e) {
- LOGGER.error(e,
- "Unable to start execution of graph " + e.getMessage());
+ } catch (KettleException | IOException e) {
+ LOGGER.error(e, "Unable to start execution of graph " + e.getMessage());
throw new DataLoadingException("Unable to start execution of graph ", e);
}
@@ -233,14 +203,13 @@ public class DataGraphExecuter {
}
/**
- * @param measureColumns
* @param stepsMeta
* @param builder
* @param measuresInCSVFile
* @throws DataLoadingException
*/
- private void processHadoopFileInputMeta(List<String> measureColumns, List<StepMeta> stepsMeta,
- StringBuilder builder, StringBuilder measuresInCSVFile) throws DataLoadingException {
+ private void processHadoopFileInputMeta(List<StepMeta> stepsMeta, StringBuilder builder,
+ StringBuilder measuresInCSVFile) throws DataLoadingException {
for (StepMeta step : stepsMeta) {
if (step.getStepMetaInterface() instanceof HadoopFileInputMeta) {
@@ -356,14 +325,13 @@ public class DataGraphExecuter {
}
/**
- * @param measureColumns
* @param stepsMeta
* @param builder
* @param measuresInCSVFile
* @throws DataLoadingException
*/
- private void processCsvInputMeta(List<String> measureColumns, List<StepMeta> stepsMeta,
- StringBuilder builder, StringBuilder measuresInCSVFile) throws DataLoadingException {
+ private void processCsvInputMeta(List<StepMeta> stepsMeta, StringBuilder builder,
+ StringBuilder measuresInCSVFile) throws DataLoadingException {
for (StepMeta step : stepsMeta) {
if (step.getStepMetaInterface() instanceof CsvInputMeta) {
if (null != model.getCsvFilePath() && model.getRddIteratorKey() == null) {
@@ -419,11 +387,10 @@ public class DataGraphExecuter {
/**
* This method will validate the both fact as well as dimension csv files.
*
- * @param schemaInfo
+ * @param schema
* @throws DataLoadingException
*/
- private void validateCSVFiles(SchemaInfo schemaInfo, String partitionId,
- CarbonDataLoadSchema schema) throws DataLoadingException {
+ private void validateCSVFiles(CarbonDataLoadSchema schema) throws DataLoadingException {
// Validate the Fact CSV Files.
String csvFilePath = model.getCsvFilePath();
if (csvFilePath != null) {
@@ -445,7 +412,7 @@ public class DataGraphExecuter {
});
for (CarbonFile f : listFiles) {
- validateCSV(schemaInfo, model.getTableName(), f, partitionId, schema, ",");
+ validateCSV(model.getTableName(), f, schema, ",");
}
} else {
@@ -458,12 +425,12 @@ public class DataGraphExecuter {
}
if (exists) {
- validateCSV(schemaInfo, model.getTableName(),
- FileFactory.getCarbonFile(csvFilePath, fileType), partitionId, schema, ",");
+ validateCSV(model.getTableName(),
+ FileFactory.getCarbonFile(csvFilePath, fileType), schema, ",");
} else {
- validateCSV(schemaInfo, model.getTableName(), FileFactory
+ validateCSV(model.getTableName(), FileFactory
.getCarbonFile(csvFilePath + CarbonCommonConstants.FILE_INPROGRESS_STATUS,
- fileType), partitionId, schema, ",");
+ fileType), schema, ",");
}
}
@@ -487,8 +454,8 @@ public class DataGraphExecuter {
try {
FileFactory.FileType fileType = FileFactory.getFileType(file);
if (FileFactory.isFileExist(file, fileType)) {
- validateCSV(schemaInfo, model.getTableName(),
- FileFactory.getCarbonFile(file, fileType), partitionId, schema,
+ validateCSV(model.getTableName(),
+ FileFactory.getCarbonFile(file, fileType), schema,
model.getCsvDelimiter());
}
} catch (IOException e) {
@@ -500,45 +467,4 @@ public class DataGraphExecuter {
}
}
- /**
- * Validate the dimension csv files.
- *
- * @param schema
- * @param schemaInfo
- * @param dimFile
- * @throws DataLoadingException
- */
- private void validateDimensionCSV(SchemaInfo schemaInfo, String factTableName,
- String dimTableName, CarbonFile dimFile, String partitionId, CarbonDataLoadSchema schema,
- String delimiter) throws DataLoadingException {
- String[] columnNames =
- getDimColumnNames(schemaInfo, factTableName, dimTableName, partitionId, schema);
-
- if (null == columnNames || columnNames.length < 1) {
- return;
- }
- if (!checkAllColumnsPresent(columnNames, dimFile.getAbsolutePath(), delimiter)) {
- LOGGER.error(
- "CSV File provided is not proper. Column names in schema and csv header are not same. "
- + "CSVFile Name : "
- + dimFile.getName());
- throw new DataLoadingException(DataProcessorConstants.CSV_VALIDATION_ERRROR_CODE,
- "Dimension CSV file provided is not proper. Column names in Schema and csv header are "
- + "not same. CSVFile Name : "
- + dimFile.getName());
- }
-
- }
-
- /**
- * Check All the columns are present in the CSV File
- *
- * @param dimFilePath
- * @return
- */
- private boolean checkAllColumnsPresent(String[] columnNames, String dimFilePath,
- String delimiter) {
- return GraphExecutionUtil.checkCSVAndRequestedTableColumns(dimFilePath, columnNames, delimiter);
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
index 2148901..64f21b0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
@@ -196,7 +196,7 @@ public final class GraphExecutionUtil {
* @return
*/
public static boolean checkCSVAndRequestedTableColumns(String csvFilePath, String[] columnNames,
- String delimiter) {
+ String delimiter) throws IOException {
String readLine = CarbonUtil.readHeader(csvFilePath);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
index 6d9be67..efe7830 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
@@ -42,7 +42,6 @@ import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.dictionary.DictionaryServerClientDictionary;
import org.apache.carbondata.processing.newflow.dictionary.DirectDictionary;
@@ -141,11 +140,7 @@ public class PrimitiveDataType implements GenericDataType<Object> {
Dictionary dictionary = null;
if (useOnePass) {
if (CarbonUtil.isFileExistsForGivenColumn(storePath, identifier)) {
- try {
- dictionary = cache.get(identifier);
- } catch (CarbonUtilException e) {
- throw new RuntimeException(e);
- }
+ dictionary = cache.get(identifier);
}
String threadNo = "initial";
DictionaryKey dictionaryKey = new DictionaryKey();
@@ -166,7 +161,7 @@ public class PrimitiveDataType implements GenericDataType<Object> {
dictionaryGenerator = new PreCreatedDictionary(dictionary);
}
}
- } catch (CarbonUtilException e) {
+ } catch (IOException e) {
throw new RuntimeException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
index a420d27..824a12e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
@@ -20,6 +20,7 @@
package org.apache.carbondata.processing.mdkeygen;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -257,8 +258,6 @@ public class MDKeyGenStep extends BaseStep {
/**
* This method will be used to get and update the step properties which will
* required to run this step
- *
- * @throws CarbonUtilException
*/
private boolean setStepConfiguration() {
this.tableName = meta.getTableName();
@@ -287,7 +286,7 @@ public class MDKeyGenStep extends BaseStep {
int[] dimLensWithComplex = null;
try {
dimLensWithComplex = CarbonUtil.getCardinalityFromLevelMetadataFile(levelCardinalityFilePath);
- } catch (CarbonUtilException e) {
+ } catch (IOException e) {
LOGGER.error("Level cardinality file :: " + e.getMessage());
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
index cc4da62..0445d84 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
@@ -19,6 +19,7 @@
package org.apache.carbondata.processing.newflow;
+import java.io.IOException;
import java.util.Iterator;
import org.apache.carbondata.common.CarbonIterator;
@@ -56,9 +57,9 @@ public abstract class AbstractDataLoadProcessorStep {
/**
* Initialization process for this step.
*
- * @throws CarbonDataLoadingException
+ * @throws IOException
*/
- public abstract void initialize() throws CarbonDataLoadingException;
+ public abstract void initialize() throws IOException;
/**
* Tranform the data as per the implementation.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/RowConverter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/RowConverter.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/RowConverter.java
index 3b199ab..1180777 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/RowConverter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/RowConverter.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.processing.newflow.converter;
+import java.io.IOException;
+
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.newflow.row.CarbonRow;
@@ -26,7 +28,7 @@ import org.apache.carbondata.processing.newflow.row.CarbonRow;
*/
public interface RowConverter {
- void initialize();
+ void initialize() throws IOException;
CarbonRow convert(CarbonRow row) throws CarbonDataLoadingException;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
index 82a7bc2..6e6243b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
@@ -19,14 +19,14 @@
package org.apache.carbondata.processing.newflow.converter.impl;
+import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.dictionary.*;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -35,7 +35,6 @@ import org.apache.carbondata.core.devapi.DictionaryGenerationException;
import org.apache.carbondata.core.dictionary.client.DictionaryClient;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryKey;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.DataField;
import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
@@ -46,9 +45,6 @@ import org.apache.carbondata.processing.newflow.row.CarbonRow;
public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConverterImpl {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DictionaryFieldConverterImpl.class.getName());
-
private BiDictionary<Integer, Object> dictionaryGenerator;
private int index;
@@ -60,7 +56,8 @@ public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConvert
public DictionaryFieldConverterImpl(DataField dataField,
Cache<DictionaryColumnUniqueIdentifier, Dictionary> cache,
CarbonTableIdentifier carbonTableIdentifier, String nullFormat, int index,
- DictionaryClient client, Boolean useOnePass, String storePath) {
+ DictionaryClient client, Boolean useOnePass, String storePath)
+ throws IOException {
this.index = index;
this.carbonDimension = (CarbonDimension) dataField.getColumn();
this.nullFormat = nullFormat;
@@ -72,12 +69,7 @@ public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConvert
// if use one pass, use DictionaryServerClientDictionary
if (useOnePass) {
if (CarbonUtil.isFileExistsForGivenColumn(storePath, identifier)) {
- try{
- dictionary = cache.get(identifier);
- } catch (CarbonUtilException e) {
- LOGGER.error(e);
- throw new RuntimeException(e);
- }
+ dictionary = cache.get(identifier);
}
String threadNo = "initial";
DictionaryKey dictionaryKey = new DictionaryKey();
@@ -94,13 +86,8 @@ public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConvert
dictionaryGenerator = new DictionaryServerClientDictionary(dictionary, client,
dictionaryKey, localCache);
} else {
- try {
- dictionary = cache.get(identifier);
- dictionaryGenerator = new PreCreatedDictionary(dictionary);
- } catch (CarbonUtilException e) {
- LOGGER.error(e);
- throw new RuntimeException(e);
- }
+ dictionary = cache.get(identifier);
+ dictionaryGenerator = new PreCreatedDictionary(dictionary);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
index b065b55..dac8593 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.processing.newflow.converter.impl;
+import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.cache.Cache;
@@ -62,7 +63,8 @@ public class FieldEncoderFactory {
public FieldConverter createFieldEncoder(DataField dataField,
Cache<DictionaryColumnUniqueIdentifier, Dictionary> cache,
CarbonTableIdentifier carbonTableIdentifier, int index, String nullFormat,
- DictionaryClient client, Boolean useOnePass, String storePath) {
+ DictionaryClient client, Boolean useOnePass, String storePath)
+ throws IOException {
// Converters are only needed for dimensions and measures it return null.
if (dataField.getColumn().isDimesion()) {
if (dataField.getColumn().hasEncoding(Encoding.DIRECT_DICTIONARY) &&
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
index e46b86a..7f7deb1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
@@ -18,9 +18,14 @@
*/
package org.apache.carbondata.processing.newflow.converter.impl;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheProvider;
@@ -67,7 +72,7 @@ public class RowConverterImpl implements RowConverter {
}
@Override
- public void initialize() {
+ public void initialize() throws IOException {
CacheProvider cacheProvider = CacheProvider.getInstance();
Cache<DictionaryColumnUniqueIdentifier, Dictionary> cache =
cacheProvider.createCache(CacheType.REVERSE_DICTIONARY,
@@ -102,10 +107,8 @@ public class RowConverterImpl implements RowConverter {
try {
dictClient = result.get();
- } catch (InterruptedException e) {
- e.printStackTrace();
- } catch (ExecutionException e) {
- e.printStackTrace();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException(e);
}
}
for (int i = 0; i < fields.length; i++) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java
index 4504f0c..73e43fd 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java
@@ -163,7 +163,7 @@ public class UnsafeParallelReadMergeSorterImpl implements Sorter {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
.recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
return false;
- } catch (CarbonSortKeyAndGroupByException e) {
+ } catch (InterruptedException e) {
throw new CarbonDataLoadingException(e);
}
}
[4/9] incubator-carbondata git commit: clean up core
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
index 943ea1a..90ef0bb 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
@@ -19,11 +19,23 @@
package org.apache.carbondata.scan.filter;
+import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -46,18 +58,25 @@ import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.ColumnExpression;
import org.apache.carbondata.scan.expression.Expression;
import org.apache.carbondata.scan.expression.ExpressionResult;
import org.apache.carbondata.scan.expression.LiteralExpression;
-import org.apache.carbondata.scan.expression.UnknownExpression;
import org.apache.carbondata.scan.expression.conditional.ListExpression;
import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.executer.*;
+import org.apache.carbondata.scan.filter.executer.AndFilterExecuterImpl;
+import org.apache.carbondata.scan.filter.executer.DimColumnExecuterFilterInfo;
+import org.apache.carbondata.scan.filter.executer.ExcludeColGroupFilterExecuterImpl;
+import org.apache.carbondata.scan.filter.executer.ExcludeFilterExecuterImpl;
+import org.apache.carbondata.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
+import org.apache.carbondata.scan.filter.executer.IncludeFilterExecuterImpl;
+import org.apache.carbondata.scan.filter.executer.OrFilterExecuterImpl;
+import org.apache.carbondata.scan.filter.executer.RestructureFilterExecuterImpl;
+import org.apache.carbondata.scan.filter.executer.RowLevelFilterExecuterImpl;
+import org.apache.carbondata.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
import org.apache.carbondata.scan.filter.intf.ExpressionType;
import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.scan.filter.intf.RowImpl;
@@ -287,7 +306,7 @@ public final class FilterUtil {
int j = 0;
for (Iterator<Integer> iterator = integers.iterator(); iterator.hasNext(); ) {
Integer integer = iterator.next();
- byteIndexs[j++] = integer.intValue();
+ byteIndexs[j++] = integer;
}
return byteIndexs;
}
@@ -296,14 +315,11 @@ public final class FilterUtil {
* This method will get the no dictionary data based on filters and same
* will be in DimColumnFilterInfo
*
- * @param tableIdentifier
- * @param columnExpression
* @param evaluateResultListFinal
* @param isIncludeFilter
* @return DimColumnFilterInfo
*/
public static DimColumnFilterInfo getNoDictionaryValKeyMemberForFilter(
- AbsoluteTableIdentifier tableIdentifier, ColumnExpression columnExpression,
List<String> evaluateResultListFinal, boolean isIncludeFilter) {
List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
for (String result : evaluateResultListFinal) {
@@ -338,11 +354,11 @@ public final class FilterUtil {
* @param evaluateResultList
* @param isIncludeFilter
* @return
- * @throws QueryExecutionException
+ * @throws IOException
*/
public static DimColumnFilterInfo getFilterValues(AbsoluteTableIdentifier tableIdentifier,
ColumnExpression columnExpression, List<String> evaluateResultList, boolean isIncludeFilter)
- throws QueryExecutionException {
+ throws IOException {
Dictionary forwardDictionary = null;
try {
// Reading the dictionary value from cache.
@@ -364,11 +380,9 @@ public final class FilterUtil {
* @param forwardDictionary
* @param isIncludeFilter
* @return
- * @throws QueryExecutionException
*/
private static DimColumnFilterInfo getFilterValues(ColumnExpression columnExpression,
- List<String> evaluateResultList, Dictionary forwardDictionary, boolean isIncludeFilter)
- throws QueryExecutionException {
+ List<String> evaluateResultList, Dictionary forwardDictionary, boolean isIncludeFilter) {
sortFilterModelMembers(columnExpression, evaluateResultList);
List<Integer> surrogates =
new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
@@ -390,10 +404,9 @@ public final class FilterUtil {
*
* @param evaluateResultList filter value
* @param surrogates
- * @throws QueryExecutionException
*/
private static void getDictionaryValue(List<String> evaluateResultList,
- Dictionary forwardDictionary, List<Integer> surrogates) throws QueryExecutionException {
+ Dictionary forwardDictionary, List<Integer> surrogates) {
((ForwardDictionary) forwardDictionary)
.getSurrogateKeyByIncrementalSearch(evaluateResultList, surrogates);
}
@@ -408,12 +421,12 @@ public final class FilterUtil {
* @param isIncludeFilter
* @return DimColumnFilterInfo
* @throws FilterUnsupportedException
- * @throws QueryExecutionException
+ * @throws IOException
*/
public static DimColumnFilterInfo getFilterListForAllValues(
AbsoluteTableIdentifier tableIdentifier, Expression expression,
final ColumnExpression columnExpression, boolean isIncludeFilter)
- throws FilterUnsupportedException {
+ throws IOException, FilterUnsupportedException {
Dictionary forwardDictionary = null;
List<String> evaluateResultListFinal = new ArrayList<String>(20);
DictionaryChunksWrapper dictionaryWrapper = null;
@@ -433,7 +446,7 @@ public final class FilterUtil {
row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue,
columnExpression.getCarbonColumn().getDataType()) });
Boolean rslt = expression.evaluate(row).getBoolean();
- if (null != rslt && !(rslt ^ isIncludeFilter)) {
+ if (null != rslt && rslt == isIncludeFilter) {
if (null == stringValue) {
evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
} else {
@@ -446,8 +459,6 @@ public final class FilterUtil {
}
return getFilterValues(columnExpression, evaluateResultListFinal, forwardDictionary,
isIncludeFilter);
- } catch (QueryExecutionException e) {
- throw new FilterUnsupportedException(e.getMessage());
} finally {
CarbonUtil.clearDictionaryCache(forwardDictionary);
}
@@ -467,49 +478,6 @@ public final class FilterUtil {
}
/**
- * Metahod will resolve the filter member to its respective surrogates by
- * scanning the dictionary cache.
- *
- * @param tableIdentifier
- * @param expression
- * @param columnExpression
- * @param isIncludeFilter
- * @return
- * @throws QueryExecutionException
- * @throws FilterUnsupportedException
- */
- public static DimColumnFilterInfo getFilterList(AbsoluteTableIdentifier tableIdentifier,
- Expression expression, ColumnExpression columnExpression, boolean isIncludeFilter)
- throws QueryExecutionException, FilterUnsupportedException {
- DimColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal = new ArrayList<String>(20);
- try {
- List<ExpressionResult> evaluateResultList = expression.evaluate(null).getList();
- for (ExpressionResult result : evaluateResultList) {
- if (result.getString() == null) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
- continue;
- }
- evaluateResultListFinal.add(result.getString());
- }
-
- if (null != columnExpression.getCarbonColumn() && !columnExpression.getCarbonColumn()
- .hasEncoding(Encoding.DICTIONARY)) {
- resolvedFilterObject =
- getNoDictionaryValKeyMemberForFilter(tableIdentifier, columnExpression,
- evaluateResultListFinal, isIncludeFilter);
- } else {
- resolvedFilterObject =
- getFilterValues(tableIdentifier, columnExpression, evaluateResultListFinal,
- isIncludeFilter);
- }
- } catch (FilterIllegalMemberException e) {
- LOGGER.audit(e.getMessage());
- }
- return resolvedFilterObject;
- }
-
- /**
* Method will prepare the dimfilterinfo instance by resolving the filter
* expression value to its respective surrogates in the scenario of restructure.
*
@@ -584,7 +552,7 @@ public final class FilterUtil {
row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(defaultValues,
columnExpression.getCarbonColumn().getDataType()) });
Boolean rslt = expression.evaluate(row).getBoolean();
- if (null != rslt && !(rslt ^ isIncludeFilter)) {
+ if (null != rslt && rslt == isIncludeFilter) {
if (null == defaultValues) {
evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
} else {
@@ -686,7 +654,7 @@ public final class FilterUtil {
* @return long[] start key
*/
public static void getStartKey(Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
- long[] startKey, List<long[]> startKeyList) throws QueryExecutionException {
+ long[] startKey, List<long[]> startKeyList) {
for(int i = 0; i < startKey.length; i++) {
// The min surrogate key is 1, set it as the init value for starkey of each column level
startKey[i] = 1;
@@ -861,13 +829,13 @@ public final class FilterUtil {
}
public static void getEndKey(Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
- AbsoluteTableIdentifier tableIdentifier, long[] endKey, SegmentProperties segmentProperties,
- List<long[]> endKeyList) throws QueryExecutionException {
+ long[] endKey, SegmentProperties segmentProperties,
+ List<long[]> endKeyList) {
List<CarbonDimension> updatedDimListBasedOnKeyGenerator =
getCarbonDimsMappedToKeyGenerator(segmentProperties.getDimensions());
for (int i = 0; i < endKey.length; i++) {
- endKey[i] = getMaxValue(tableIdentifier, updatedDimListBasedOnKeyGenerator.get(i),
+ endKey[i] = getMaxValue(updatedDimListBasedOnKeyGenerator.get(i),
segmentProperties.getDimColumnsCardinality());
}
getEndKeyWithFilter(dimensionFilter, endKey, endKeyList);
@@ -925,13 +893,8 @@ public final class FilterUtil {
* determining the end key of particular btree.
*
* @param dimCarinality
- * @throws QueryExecutionException
*/
- private static long getMaxValue(AbsoluteTableIdentifier tableIdentifier,
- CarbonDimension carbonDimension, int[] dimCarinality) throws QueryExecutionException {
- // if (DataType.TIMESTAMP == carbonDimension.getDataType()) {
- // return Integer.MAX_VALUE;
- // }
+ private static long getMaxValue(CarbonDimension carbonDimension, int[] dimCarinality) {
// Get data from all the available slices of the table
if (null != dimCarinality) {
return dimCarinality[carbonDimension.getKeyOrdinal()];
@@ -943,10 +906,9 @@ public final class FilterUtil {
* @param tableIdentifier
* @param carbonDimension
* @return
- * @throws QueryExecutionException
*/
public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier,
- CarbonDimension carbonDimension) throws QueryExecutionException {
+ CarbonDimension carbonDimension) throws IOException {
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(),
carbonDimension.getColumnIdentifier(), carbonDimension.getDataType());
@@ -954,13 +916,7 @@ public final class FilterUtil {
Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache =
cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, tableIdentifier.getStorePath());
// get the forward dictionary object
- Dictionary forwardDictionary;
- try {
- forwardDictionary = forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
- } catch (CarbonUtilException e) {
- throw new QueryExecutionException(e);
- }
- return forwardDictionary;
+ return forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
}
public static IndexKey createIndexKeyFromResolvedFilterVal(long[] startOrEndKey,
@@ -1150,14 +1106,11 @@ public final class FilterUtil {
* utilized visitor pattern inorder to populate the start and end key population.
*
* @param segmentProperties
- * @param tableIdentifier
* @param filterResolver
* @param listOfStartEndKeys
- * @throws QueryExecutionException
*/
public static void traverseResolverTreeAndGetStartAndEndKey(SegmentProperties segmentProperties,
- AbsoluteTableIdentifier tableIdentifier, FilterResolverIntf filterResolver,
- List<IndexKey> listOfStartEndKeys) throws QueryExecutionException {
+ FilterResolverIntf filterResolver, List<IndexKey> listOfStartEndKeys) {
IndexKey searchStartKey = null;
IndexKey searchEndKey = null;
long[] startKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
@@ -1172,8 +1125,8 @@ public final class FilterUtil {
SortedMap<Integer, byte[]> defaultEndValues = new TreeMap<Integer, byte[]>();
List<long[]> startKeyList = new ArrayList<long[]>();
List<long[]> endKeyList = new ArrayList<long[]>();
- traverseResolverTreeAndPopulateStartAndEndKeys(filterResolver, tableIdentifier,
- segmentProperties, startKey, setOfStartKeyByteArray, endKey, setOfEndKeyByteArray,
+ traverseResolverTreeAndPopulateStartAndEndKeys(filterResolver, segmentProperties, startKey,
+ setOfStartKeyByteArray, endKey, setOfEndKeyByteArray,
startKeyList, endKeyList);
if (endKeyList.size() > 0) {
//get the new end key from list
@@ -1328,22 +1281,21 @@ public final class FilterUtil {
}
private static void traverseResolverTreeAndPopulateStartAndEndKeys(
- FilterResolverIntf filterResolverTree, AbsoluteTableIdentifier tableIdentifier,
- SegmentProperties segmentProperties, long[] startKeys,
+ FilterResolverIntf filterResolverTree, SegmentProperties segmentProperties, long[] startKeys,
SortedMap<Integer, byte[]> setOfStartKeyByteArray, long[] endKeys,
SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> startKeyList,
- List<long[]> endKeyList) throws QueryExecutionException {
+ List<long[]> endKeyList) {
if (null == filterResolverTree) {
return;
}
- traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getLeft(), tableIdentifier,
+ traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getLeft(),
segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray,
startKeyList, endKeyList);
filterResolverTree.getStartKey(startKeys, setOfStartKeyByteArray, startKeyList);
- filterResolverTree.getEndKey(segmentProperties, tableIdentifier, endKeys, setOfEndKeyByteArray,
+ filterResolverTree.getEndKey(segmentProperties, endKeys, setOfEndKeyByteArray,
endKeyList);
- traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getRight(), tableIdentifier,
+ traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getRight(),
segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray,
startKeyList, endKeyList);
}
@@ -1377,24 +1329,11 @@ public final class FilterUtil {
*/
public static void logError(Throwable e, boolean invalidRowsPresent) {
if (!invalidRowsPresent) {
- invalidRowsPresent = true;
LOGGER.error(e, CarbonCommonConstants.FILTER_INVALID_MEMBER + e.getMessage());
}
}
/**
- * This method will return list of all the unknown expressions
- *
- * @param expression
- */
- public static List<UnknownExpression> getUnknownExpressionsList(Expression expression) {
- List<UnknownExpression> listOfExp =
- new ArrayList<UnknownExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- getUnknownExpressionsList(expression, listOfExp);
- return listOfExp;
- }
-
- /**
* This method will compare double values for its equality and also it will preserve
* the -0.0 and 0.0 equality as per == ,also preserve NaN equality check as per
* java.lang.Double.equals()
@@ -1411,21 +1350,4 @@ public final class FilterUtil {
}
- /**
- * This method will prepare the list with all unknown expressions
- *
- * @param expression
- * @param lst
- */
- private static void getUnknownExpressionsList(Expression expression,
- List<UnknownExpression> lst) {
- if (expression instanceof UnknownExpression) {
- UnknownExpression colExp = (UnknownExpression) expression;
- lst.add(colExp);
- return;
- }
- for (Expression child : expression.getChildren()) {
- getUnknownExpressionsList(child, lst);
- }
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/GenericQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/GenericQueryType.java b/core/src/main/java/org/apache/carbondata/scan/filter/GenericQueryType.java
index 881489c..bcf0dc3 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/GenericQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/GenericQueryType.java
@@ -22,7 +22,6 @@ package org.apache.carbondata.scan.filter;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.List;
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.scan.processor.BlocksChunkHolder;
@@ -39,35 +38,16 @@ public interface GenericQueryType {
void setParentname(String parentname);
- int getBlockIndex();
-
- void setBlockIndex(int blockIndex);
-
void addChildren(GenericQueryType children);
- void getAllPrimitiveChildren(List<GenericQueryType> primitiveChild);
-
- int getSurrogateIndex();
-
- void setSurrogateIndex(int surrIndex);
-
int getColsCount();
- void setKeySize(int[] keyBlockSize);
-
- int getKeyOrdinalForQuery();
-
- void setKeyOrdinalForQuery(int keyOrdinalForQuery);
-
void parseBlocksAndReturnComplexColumnByteArray(DimensionColumnDataChunk[] dimensionDataChunks,
int rowNumber, DataOutputStream dataOutputStream) throws IOException;
DataType getSchemaType();
- void parseAndGetResultBytes(ByteBuffer complexData, DataOutputStream dataOutput)
- throws IOException;
-
- void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder);
+ void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) throws IOException;
Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/AndFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/AndFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/AndFilterExecuterImpl.java
index cce4fe2..ff4f8ce 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/AndFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/AndFilterExecuterImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.executer;
+import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
@@ -34,7 +35,7 @@ public class AndFilterExecuterImpl implements FilterExecuter {
}
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException {
+ throws FilterUnsupportedException, IOException {
BitSet leftFilters = leftExecuter.applyFilter(blockChunkHolder);
if (leftFilters.isEmpty()) {
return leftFilters;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
index 4ef2c15..2b006dc 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -95,12 +95,10 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl
mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(), colGrpId));
int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- int[] maksedByte = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
KeyStructureInfo restructureInfos = new KeyStructureInfo();
restructureInfos.setKeyGenerator(keyGenerator);
restructureInfos.setMaskByteRanges(maskByteRanges);
restructureInfos.setMaxKey(maxKey);
- restructureInfos.setMaskedBytes(maksedByte);
return restructureInfos;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
index db4326b..b3136b6 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.executer;
+import java.io.IOException;
import java.util.BitSet;
import java.util.List;
@@ -46,7 +47,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) {
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/FilterExecuter.java
index 29a72da..8d03093 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/FilterExecuter.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/FilterExecuter.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.executer;
+import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
@@ -31,7 +32,8 @@ public interface FilterExecuter {
* @return
* @throws FilterUnsupportedException
*/
- BitSet applyFilter(BlocksChunkHolder blocksChunkHolder) throws FilterUnsupportedException;
+ BitSet applyFilter(BlocksChunkHolder blocksChunkHolder)
+ throws FilterUnsupportedException, IOException;
/**
* API will verify whether the block can be shortlisted based on block
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
index edceb82..57b1f4d 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
@@ -95,12 +95,10 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
mdKeyOrdinal.add(getMdkeyOrdinal(dimColumnEvaluatorInfo.getColumnIndex(), colGrpId));
int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- int[] maksedByte = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
KeyStructureInfo restructureInfos = new KeyStructureInfo();
restructureInfos.setKeyGenerator(keyGenerator);
restructureInfos.setMaskByteRanges(maskByteRanges);
restructureInfos.setMaxKey(maxKey);
- restructureInfos.setMaskedBytes(maksedByte);
return restructureInfos;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
index f399aa9..becec63 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.executer;
+import java.io.IOException;
import java.util.BitSet;
import java.util.List;
@@ -42,13 +43,12 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
this.segmentProperties = segmentProperties;
dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
- FilterUtil
- .prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(), segmentProperties,
- dimColumnEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
+ FilterUtil.prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(),
+ segmentProperties, dimColumnEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) {
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/OrFilterExecuterImpl.java
index 8fd5431..1ca89b1 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/OrFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/OrFilterExecuterImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.executer;
+import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
@@ -34,7 +35,7 @@ public class OrFilterExecuterImpl implements FilterExecuter {
}
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException {
+ throws FilterUnsupportedException, IOException {
BitSet leftFilters = leftExecuter.applyFilter(blockChunkHolder);
BitSet rightFilters = rightExecuter.applyFilter(blockChunkHolder);
leftFilters.or(rightFilters);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 00f80bb..678cf51 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -42,7 +42,6 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.scan.executor.util.QueryUtil;
import org.apache.carbondata.scan.expression.Expression;
@@ -94,7 +93,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException {
+ throws FilterUnsupportedException, IOException {
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
@@ -112,8 +111,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
// CHECKSTYLE:OFF Approval No:Approval-V1R2C10_001
if (null != msrColEvalutorInfoList) {
for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
- if (msrColumnEvalutorInfo.isMeasureExistsInCurrentSlice() && null == blockChunkHolder
- .getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]) {
+ if (null == blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()])
+ {
blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()] =
blockChunkHolder.getDataBlock().getMeasureChunk(blockChunkHolder.getFileReader(),
msrColumnEvalutorInfo.getColumnIndex());
@@ -127,11 +126,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
RowIntf row = new RowImpl();
boolean invalidRowsPresent = false;
for (int index = 0; index < numberOfRows; index++) {
- try {
- createRow(blockChunkHolder, row, index);
- } catch (QueryExecutionException e) {
- FilterUtil.logError(e, invalidRowsPresent);
- }
+ createRow(blockChunkHolder, row, index);
Boolean rslt = false;
try {
rslt = exp.evaluate(row).getBoolean();
@@ -156,12 +151,12 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @param blockChunkHolder
* @param row
* @param index
- * @throws QueryExecutionException
+ * @throws IOException
*/
private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int index)
- throws QueryExecutionException {
+ throws IOException {
Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
- String memberString = null;
+ String memberString;
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
@@ -178,7 +173,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
.getDimensionDataChunk()[blocksIndex[i]];
if (null != dimensionColumnDataChunk.getCompleteDataChunk()) {
memberString =
- readMemberBasedOnNoDictionaryVal(dimColumnEvaluatorInfo, dimensionColumnDataChunk,
+ readMemberBasedOnNoDictionaryVal(dimensionColumnDataChunk,
index);
if (null != memberString) {
if (memberString.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
@@ -195,12 +190,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
int dictionaryValue =
readSurrogatesFromColumnBlock(blockChunkHolder, index, dimColumnEvaluatorInfo,
blocksIndex[i]);
- Dictionary forwardDictionary = null;
if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
&& !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
memberString =
- getFilterActualValueFromDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue,
- forwardDictionary);
+ getFilterActualValueFromDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
.getDataBasedOnDataType(memberString,
dimColumnEvaluatorInfo.getDimension().getDataType());
@@ -244,32 +237,27 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
msrType = DataType.DOUBLE;
}
// if measure doesnt exist then set the default value.
- if (!msrColumnEvalutorInfo.isMeasureExistsInCurrentSlice()) {
- record[msrColumnEvalutorInfo.getRowIndex()] = msrColumnEvalutorInfo.getDefaultValue();
- } else {
- Object msrValue;
- switch (msrType) {
- case INT:
- case LONG:
- msrValue =
- blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getMeasureDataHolder().getReadableLongValueByIndex(index);
- break;
- case DECIMAL:
- msrValue =
- blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
- break;
- default:
- msrValue =
- blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getMeasureDataHolder().getReadableDoubleValueByIndex(index);
- }
- record[msrColumnEvalutorInfo.getRowIndex()] =
- blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getNullValueIndexHolder().getBitSet().get(index) ? null : msrValue;
-
+ Object msrValue;
+ switch (msrType) {
+ case INT:
+ case LONG:
+ msrValue =
+ blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ .getMeasureDataHolder().getReadableLongValueByIndex(index);
+ break;
+ case DECIMAL:
+ msrValue =
+ blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ .getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
+ break;
+ default:
+ msrValue =
+ blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ .getMeasureDataHolder().getReadableDoubleValueByIndex(index);
}
+ record[msrColumnEvalutorInfo.getRowIndex()] =
+ blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ .getNullValueIndexHolder().getBitSet().get(index) ? null : msrValue;
}
row.setValues(record);
}
@@ -299,20 +287,14 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
*
* @param dimColumnEvaluatorInfo
* @param dictionaryValue
- * @param forwardDictionary
* @return
- * @throws QueryExecutionException
+ * @throws IOException
*/
private String getFilterActualValueFromDictionaryValue(
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue,
- Dictionary forwardDictionary) throws QueryExecutionException {
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) throws IOException {
String memberString;
- try {
- forwardDictionary = FilterUtil
- .getForwardDictionaryCache(tableIdentifier, dimColumnEvaluatorInfo.getDimension());
- } catch (QueryExecutionException e) {
- throw new QueryExecutionException(e);
- }
+ Dictionary forwardDictionary = FilterUtil.getForwardDictionaryCache(
+ tableIdentifier, dimColumnEvaluatorInfo.getDimension());
memberString = forwardDictionary.getDictionaryValueForKey(dictionaryValue);
if (null != memberString) {
@@ -375,13 +357,11 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* directly the filter data will read, no need to scan the dictionary
* or read the dictionary value.
*
- * @param dimColumnEvaluatorInfo
* @param dimensionColumnDataChunk
* @param index
* @return
*/
private String readMemberBasedOnNoDictionaryVal(
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
VariableLengthDimensionDataChunk dimensionColumnDataChunk, int index) {
byte[] noDictionaryVals;
if (null != dimensionColumnDataChunk.getAttributes().getInvertedIndexesReverse()) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 10b83d7..cb21c88 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.executer;
+import java.io.IOException;
import java.util.BitSet;
import java.util.List;
@@ -73,7 +74,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
}
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException {
+ throws FilterUnsupportedException, IOException {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
return super.applyFilter(blockChunkHolder);
}
@@ -120,8 +121,8 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
.getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
filterValues[i], true);
if (start >= 0) {
- start = CarbonUtil.nextGreaterValueToTarget(start,
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, filterValues[i], numerOfRows);
+ start = CarbonUtil.nextGreaterValueToTarget(start, dimensionColumnDataChunk,
+ filterValues[i], numerOfRows);
}
// Logic will handle the case where the range filter member is not present in block
// in this case the binary search will return the index from where the bit sets will be
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index 808cb51..7f4cc7c 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.executer;
+import java.io.IOException;
import java.util.BitSet;
import java.util.List;
@@ -74,7 +75,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
}
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException {
+ throws FilterUnsupportedException, IOException {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
return super.applyFilter(blockChunkHolder);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 367ebbd..4d7284d 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.executer;
+import java.io.IOException;
import java.util.BitSet;
import java.util.List;
@@ -76,7 +77,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
}
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException {
+ throws FilterUnsupportedException, IOException {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
return super.applyFilter(blockChunkHolder);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 31760e6..4819cb5 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.executer;
+import java.io.IOException;
import java.util.BitSet;
import java.util.List;
@@ -76,7 +77,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
}
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException {
+ throws FilterUnsupportedException, IOException {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
return super.applyFilter(blockChunkHolder);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
index 4c46f6d..cef0f06 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
@@ -35,7 +35,6 @@ public class RowLevelRangeTypeExecuterFacory {
*
* @param filterExpressionResolverTree
* @param segmentProperties
- * @param dataType DataType
* @return the generator instance
*/
public static RowLevelFilterExecuterImpl getRowLevelRangeTypeExecuter(
@@ -49,7 +48,7 @@ public class RowLevelRangeTypeExecuterFacory {
.getDimColEvaluatorInfoList(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
.getMsrColEvalutorInfoList(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getFilterExpression(),
+ filterExpressionResolverTree.getFilterExpression(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
.getFilterRangeValues(segmentProperties), segmentProperties);
@@ -59,7 +58,7 @@ public class RowLevelRangeTypeExecuterFacory {
.getDimColEvaluatorInfoList(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
.getMsrColEvalutorInfoList(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getFilterExpression(),
+ filterExpressionResolverTree.getFilterExpression(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
.getFilterRangeValues(segmentProperties), segmentProperties);
@@ -69,7 +68,7 @@ public class RowLevelRangeTypeExecuterFacory {
.getDimColEvaluatorInfoList(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
.getMsrColEvalutorInfoList(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getFilterExpression(),
+ filterExpressionResolverTree.getFilterExpression(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
.getFilterRangeValues(segmentProperties), segmentProperties);
@@ -79,7 +78,7 @@ public class RowLevelRangeTypeExecuterFacory {
.getDimColEvaluatorInfoList(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
.getMsrColEvalutorInfoList(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getFilterExpression(),
+ filterExpressionResolverTree.getFilterExpression(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
.getFilterRangeValues(segmentProperties), segmentProperties);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/AndFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
deleted file mode 100644
index 2495621..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver;
-
-import java.util.List;
-import java.util.SortedMap;
-
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.scan.expression.BinaryExpression;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-
-public class AndFilterResolverImpl extends LogicalFilterResolverImpl {
-
- /**
- *i
- */
- private static final long serialVersionUID = -761688076874662001L;
-
- public AndFilterResolverImpl(FilterResolverIntf leftEvalutor, FilterResolverIntf rightEvalutor,
- ExpressionType filterExpressionType,BinaryExpression expression) {
- super(leftEvalutor, rightEvalutor, expression);
- }
-
- @Override public void getStartKey(long[] startKeys,
- SortedMap<Integer, byte[]> noDicStartKeys, List<long[]> startKeyList)
- throws QueryExecutionException {
- leftEvalutor.getStartKey(startKeys, noDicStartKeys, startKeyList);
- rightEvalutor.getStartKey(startKeys, noDicStartKeys, startKeyList);
- }
-
- @Override public void getEndKey(SegmentProperties segmentProperties,
- AbsoluteTableIdentifier tableIdentifier, long[] endKeys,
- SortedMap<Integer, byte[]> noDicEndKeys, List<long[]> endKeyList)
- throws QueryExecutionException {
- leftEvalutor.getEndKey(segmentProperties, tableIdentifier, endKeys, noDicEndKeys, endKeyList);
- rightEvalutor.getEndKey(segmentProperties, tableIdentifier, endKeys, noDicEndKeys, endKeyList);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
index b8e0b79..d22bf38 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter.resolver;
+import java.io.IOException;
import java.util.List;
import java.util.SortedMap;
@@ -26,7 +27,6 @@ import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.ColumnExpression;
import org.apache.carbondata.scan.expression.Expression;
import org.apache.carbondata.scan.expression.conditional.BinaryConditionalExpression;
@@ -59,11 +59,10 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
* dictionaries for executing/evaluating the filter expressions in the
* executer layer.
*
- * @throws QueryExecutionException
* @throws FilterUnsupportedException
*/
@Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
- throws FilterUnsupportedException {
+ throws FilterUnsupportedException, IOException {
FilterResolverMetadata metadata = new FilterResolverMetadata();
metadata.setTableIdentifier(absoluteTableIdentifier);
if ((!isExpressionResolve) && exp instanceof BinaryConditionalExpression) {
@@ -189,32 +188,22 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
* method will calculates the start key based on the filter surrogates
*/
public void getStartKey(long[] startKey,
- SortedMap<Integer, byte[]> setOfStartKeyByteArray, List<long[]> startKeyList)
- throws QueryExecutionException {
- if (null == dimColResolvedFilterInfo.getStarIndexKey()) {
- FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
- startKey, startKeyList);
- FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
- setOfStartKeyByteArray);
- }
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray, List<long[]> startKeyList) {
+ FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
+ startKey, startKeyList);
+ FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
+ setOfStartKeyByteArray);
}
/**
- * method will get the start key based on the filter surrogates
- *
- * @return end IndexKey
- * @throws QueryExecutionException
+ * get the start key based on the filter surrogates
*/
- @Override public void getEndKey(SegmentProperties segmentProperties,
- AbsoluteTableIdentifier absoluteTableIdentifier, long[] endKeys,
- SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList)
- throws QueryExecutionException {
- if (null == dimColResolvedFilterInfo.getEndIndexKey()) {
- FilterUtil.getEndKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
- absoluteTableIdentifier, endKeys, segmentProperties, endKeyList);
- FilterUtil.getEndKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
- setOfEndKeyByteArray);
- }
+ @Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList) {
+ FilterUtil.getEndKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(), endKeys,
+ segmentProperties, endKeyList);
+ FilterUtil.getEndKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
+ setOfEndKeyByteArray);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/FilterResolverIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/FilterResolverIntf.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/FilterResolverIntf.java
index cd108fb..dd6c5c8 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/FilterResolverIntf.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/FilterResolverIntf.java
@@ -18,13 +18,13 @@
*/
package org.apache.carbondata.scan.filter.resolver;
+import java.io.IOException;
import java.io.Serializable;
import java.util.List;
import java.util.SortedMap;
import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.Expression;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
@@ -37,10 +37,11 @@ public interface FilterResolverIntf extends Serializable {
* dictionaries for executing/evaluating the filter expressions in the
* executer layer.
*
- * @throws QueryExecutionException
+ * @throws IOException
* @throws FilterUnsupportedException
*/
- void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) throws FilterUnsupportedException;
+ void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
+ throws IOException, FilterUnsupportedException;
/**
* This API will provide the left column filter expression
@@ -69,25 +70,22 @@ public interface FilterResolverIntf extends Serializable {
/**
* API will get the start key based on the filter applied based on the key generator
*
- * @param segmentProperties
* @param startKey
* @param setOfStartKeyByteArray
*/
void getStartKey(long[] startKey, SortedMap<Integer, byte[]> setOfStartKeyByteArray,
- List<long[]> startKeyList) throws QueryExecutionException;
+ List<long[]> startKeyList);
/**
* API will read the end key based on the max surrogate of
* particular dimension column
*
- * @param setOfEndKeyByteArray
* @param endKeys
+ * @param setOfEndKeyByteArray
* @return
- * @throws QueryExecutionException
*/
- void getEndKey(SegmentProperties segmentProperties, AbsoluteTableIdentifier tableIdentifier,
- long[] endKeys, SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList)
- throws QueryExecutionException;
+ void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList);
/**
* API will return the filter executer type which will be used to evaluate
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
index 3ab3e25..4d09e65 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
@@ -24,7 +24,6 @@ import java.util.SortedMap;
import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.BinaryExpression;
import org.apache.carbondata.scan.expression.Expression;
import org.apache.carbondata.scan.filter.intf.ExpressionType;
@@ -89,14 +88,12 @@ public class LogicalFilterResolverImpl implements FilterResolverIntf {
@Override
public void getStartKey(long[] startKey, SortedMap<Integer, byte[]> setOfStartKeyByteArray,
- List<long[]> startKeyList) throws QueryExecutionException {
+ List<long[]> startKeyList) {
}
- @Override public void getEndKey(SegmentProperties segmentProperties,
- AbsoluteTableIdentifier tableIdentifier, long[] endKeys,
- SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList)
- throws QueryExecutionException {
+ @Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList) {
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RestructureFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RestructureFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RestructureFilterResolverImpl.java
deleted file mode 100644
index 15c015a..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RestructureFilterResolverImpl.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.resolver;
-
-import java.util.List;
-import java.util.SortedMap;
-
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.conditional.BinaryConditionalExpression;
-import org.apache.carbondata.scan.expression.conditional.ConditionalExpression;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-public class RestructureFilterResolverImpl implements FilterResolverIntf {
- /**
- *
- */
- private static final long serialVersionUID = -5399656036192814524L;
-
- protected DimColumnResolvedFilterInfo dimColumnResolvedFilterInfo;
-
- private Expression exp;
-
- private String defaultValue;
-
- private int surrogate;
-
- private boolean isExpressionResolve;
-
- private boolean isIncludeFilter;
-
- public RestructureFilterResolverImpl(Expression exp, String defaultValue, int surrogate,
- boolean isExpressionResolve, boolean isIncludeFilter) {
- dimColumnResolvedFilterInfo = new DimColumnResolvedFilterInfo();
- this.exp = exp;
- this.defaultValue = defaultValue;
- this.surrogate = surrogate;
- this.isExpressionResolve = isExpressionResolve;
- this.isIncludeFilter = isIncludeFilter;
- }
-
- /**
- * Method will resolve the filters and it will replace the newly added dimension with default
- * value
- *
- * @param absoluteTableIdentifier
- * @throws FilterUnsupportedException
- */
- @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
- throws FilterUnsupportedException {
-
- DimColumnResolvedFilterInfo dimColumnResolvedFilterInfo = new DimColumnResolvedFilterInfo();
- if (!this.isExpressionResolve && exp instanceof BinaryConditionalExpression) {
- BinaryConditionalExpression binaryConditionalExpression = (BinaryConditionalExpression) exp;
- Expression left = binaryConditionalExpression.getLeft();
- Expression right = binaryConditionalExpression.getRight();
- if (left instanceof ColumnExpression) {
- ColumnExpression columnExpression = (ColumnExpression) left;
- if (columnExpression.getDataType().equals(DataType.TIMESTAMP) ||
- columnExpression.getDataType().equals(DataType.DATE) ) {
- isExpressionResolve = true;
- } else {
- // If imei=imei comes in filter condition then we need to
- // skip processing of right expression.
- // This flow has reached here assuming that this is a single
- // column expression.
- // we need to check if the other expression contains column
- // expression or not in depth.
- if (FilterUtil.checkIfExpressionContainsColumn(right)) {
- isExpressionResolve = true;
- } else {
- dimColumnResolvedFilterInfo
- .setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
- dimColumnResolvedFilterInfo.setFilterValues(
- FilterUtil.getFilterListForRS(right, columnExpression, defaultValue, surrogate));
- }
- }
- } else if (right instanceof ColumnExpression) {
- ColumnExpression columnExpression = (ColumnExpression) right;
- if (columnExpression.getDataType().equals(DataType.TIMESTAMP) ||
- columnExpression.getDataType().equals(DataType.DATE)) {
- isExpressionResolve = true;
- } else {
-
- // If imei=imei comes in filter condition then we need to
- // skip processing of right expression.
- // This flow has reached here assuming that this is a single
- // column expression.
- // we need to check if the other expression contains column
- // expression or not in depth.
- if (checkIfExpressionContainsColumn(left)) {
- isExpressionResolve = true;
- } else {
- dimColumnResolvedFilterInfo
- .setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
- dimColumnResolvedFilterInfo.setFilterValues(
- FilterUtil.getFilterListForRS(left, columnExpression, defaultValue, surrogate));
- }
- }
- }
- }
- if (this.isExpressionResolve && exp instanceof ConditionalExpression) {
- ConditionalExpression conditionalExpression = (ConditionalExpression) exp;
- List<ColumnExpression> columnList = conditionalExpression.getColumnList();
- dimColumnResolvedFilterInfo.setColumnIndex(columnList.get(0).getDimension().getOrdinal());
- dimColumnResolvedFilterInfo.setFilterValues(FilterUtil
- .getFilterListForAllMembersRS(exp, columnList.get(0), defaultValue, surrogate,
- isIncludeFilter));
- }
-
- }
-
- /**
- * This method will check if a given expression contains a column expression recursively.
- *
- * @return boolean
- */
- private boolean checkIfExpressionContainsColumn(Expression expression) {
- if (expression instanceof ColumnExpression) {
- return true;
- }
- for (Expression child : expression.getChildren()) {
- if (checkIfExpressionContainsColumn(child)) {
- return true;
- }
- }
-
- return false;
- }
-
- @Override public FilterResolverIntf getLeft() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override public FilterResolverIntf getRight() {
- // TODO Auto-generated method stub
- return null;
- }
-
- /**
- * Method will return the DimColumnResolvedFilterInfo instance which consists
- * the mapping of the respective dimension and its surrogates involved in
- * filter expression.
- *
- * @return DimColumnResolvedFilterInfo
- */
- public DimColumnResolvedFilterInfo getDimColResolvedFilterInfo() {
- return dimColumnResolvedFilterInfo;
- }
-
- /**
- * For restructure resolver no implementation is required for getting
- * the start key since it already has default values
- */
- @Override public void getStartKey(long[] startKeys,
- SortedMap<Integer, byte[]> noDicStartKeys, List<long[]> startKeyList) {
-
- }
-
- /**
- * For restructure resolver no implementation is required for getting
- * the end key since it already has default values
- *
- * @return IndexKey.
- */
- @Override public void getEndKey(SegmentProperties segmentProperties,
- AbsoluteTableIdentifier tableIdentifier, long[] endKeys,
- SortedMap<Integer, byte[]> noDicEndKeys, List<long[]> endKeyList) {
- }
-
- /**
- * Method will get the executer type inorder to create filter executer tree
- *
- * @return FilterExecuterType
- */
- @Override public FilterExecuterType getFilterExecuterType() {
- return FilterExecuterType.RESTRUCTURE;
- }
-
- @Override public Expression getFilterExpression() {
- // TODO Auto-generated method stub
- return exp;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelFilterResolverImpl.java
index 466d048..e6fb166 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelFilterResolverImpl.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.scan.expression.ColumnExpression;
import org.apache.carbondata.scan.expression.Expression;
@@ -35,8 +34,6 @@ import org.apache.carbondata.scan.filter.resolver.resolverinfo.MeasureColumnReso
public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
private static final long serialVersionUID = 176122729713729929L;
- protected boolean isExpressionResolve;
- protected boolean isIncludeFilter;
private List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
private List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
@@ -74,10 +71,8 @@ public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
} else {
msrColumnEvalutorInfo = new MeasureColumnResolvedFilterInfo();
msrColumnEvalutorInfo.setRowIndex(index++);
- msrColumnEvalutorInfo.setAggregator(
- ((CarbonMeasure) columnExpression.getCarbonColumn()).getAggregateFunction());
msrColumnEvalutorInfo
- .setColumnIndex(((CarbonMeasure) columnExpression.getCarbonColumn()).getOrdinal());
+ .setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
msrColumnEvalutorInfo.setType(columnExpression.getCarbonColumn().getDataType());
msrColEvalutorInfoList.add(msrColumnEvalutorInfo);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index f07fe98..66952f7 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -25,17 +25,13 @@ import java.util.Comparator;
import java.util.List;
import java.util.SortedMap;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.ColumnExpression;
import org.apache.carbondata.scan.expression.Expression;
import org.apache.carbondata.scan.expression.ExpressionResult;
@@ -55,8 +51,6 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
*
*/
private static final long serialVersionUID = 6629319265336666789L;
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(RowLevelRangeFilterResolverImpl.class.getName());
private List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
private List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
private AbsoluteTableIdentifier tableIdentifier;
@@ -101,16 +95,10 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
*/
public void getStartKey(long[] startKey,
SortedMap<Integer, byte[]> noDictStartKeys, List<long[]> startKeyList) {
- if (null == dimColEvaluatorInfoList.get(0).getStarIndexKey()) {
- try {
- FilterUtil.getStartKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(),
- startKey, startKeyList);
- FilterUtil
- .getStartKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDictStartKeys);
- } catch (QueryExecutionException e) {
- LOGGER.error("Can not get the start key during block prune");
- }
- }
+ FilterUtil.getStartKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(),
+ startKey, startKeyList);
+ FilterUtil
+ .getStartKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDictStartKeys);
}
/**
@@ -118,20 +106,12 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
*
* @return end IndexKey
*/
- @Override public void getEndKey(SegmentProperties segmentProperties,
- AbsoluteTableIdentifier absoluteTableIdentifier, long[] endKeys,
+ @Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
SortedMap<Integer, byte[]> noDicEndKeys, List<long[]> endKeyList) {
- if (null == dimColEvaluatorInfoList.get(0).getEndIndexKey()) {
- try {
- FilterUtil.getEndKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(),
- absoluteTableIdentifier, endKeys, segmentProperties, endKeyList);
- FilterUtil
- .getEndKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDicEndKeys);
- } catch (QueryExecutionException e) {
- // TODO Auto-generated catch block
- LOGGER.error("Can not get the end key during block prune");
- }
- }
+ FilterUtil.getEndKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(),
+ endKeys, segmentProperties, endKeyList);
+ FilterUtil
+ .getEndKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDicEndKeys);
}
private List<byte[]> getNoDictionaryRangeValues() {
@@ -170,7 +150,8 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
* Method which will resolve the filter expression by converting the filter
* member to its assigned dictionary values.
*/
- public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) {
+ public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
+ throws FilterUnsupportedException {
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = null;
MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = null;
int index = 0;
@@ -182,7 +163,6 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
DimColumnFilterInfo filterInfo = new DimColumnFilterInfo();
dimColumnEvaluatorInfo.setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
- //dimColumnEvaluatorInfo.se
dimColumnEvaluatorInfo.setRowIndex(index++);
dimColumnEvaluatorInfo.setDimension(columnExpression.getDimension());
dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
@@ -199,10 +179,8 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
} else {
msrColumnEvalutorInfo = new MeasureColumnResolvedFilterInfo();
msrColumnEvalutorInfo.setRowIndex(index++);
- msrColumnEvalutorInfo.setAggregator(
- ((CarbonMeasure) columnExpression.getCarbonColumn()).getAggregateFunction());
msrColumnEvalutorInfo
- .setColumnIndex(((CarbonMeasure) columnExpression.getCarbonColumn()).getOrdinal());
+ .setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
msrColumnEvalutorInfo.setType(columnExpression.getCarbonColumn().getDataType());
msrColEvalutorInfoList.add(msrColumnEvalutorInfo);
}
@@ -210,7 +188,8 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
}
}
- private List<Integer> getDirectSurrogateValues(ColumnExpression columnExpression) {
+ private List<Integer> getDirectSurrogateValues(ColumnExpression columnExpression)
+ throws FilterUnsupportedException {
List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
@@ -229,7 +208,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
}
} catch (FilterIllegalMemberException e) {
- new FilterUnsupportedException(e);
+ throw new FilterUnsupportedException(e);
}
return filterValuesList;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
index bf61d85..4c0eae6 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
@@ -19,21 +19,20 @@
package org.apache.carbondata.scan.filter.resolver.resolverinfo;
+import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.carbon.datastore.IndexKey;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.scan.filter.DimColumnFilterInfo;
import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.visitable.ResolvedFilterInfoVisitable;
import org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor.ResolvedFilterInfoVisitorIntf;
-public class DimColumnResolvedFilterInfo implements Serializable, ResolvedFilterInfoVisitable {
+public class DimColumnResolvedFilterInfo implements Serializable {
/**
*
*/
@@ -45,35 +44,17 @@ public class DimColumnResolvedFilterInfo implements Serializable, ResolvedFilter
private int columnIndex = -1;
/**
- * need compressed data from file
- */
- private boolean needCompressedData;
-
- /**
* rowIndex
*/
private int rowIndex = -1;
private boolean isDimensionExistsInCurrentSilce = true;
- private int rsSurrogates;
-
private String defaultValue;
private CarbonDimension dimension;
/**
- * start index key of the block based on the keygenerator
- */
- private transient IndexKey starIndexKey;
-
- /**
- * end index key which is been formed considering the max surrogate values
- * from dictionary cache
- */
- private transient IndexKey endIndexKey;
-
- /**
* reolved filter object of a particlar filter Expression.
*/
private DimColumnFilterInfo resolvedFilterValueObj;
@@ -84,22 +65,6 @@ public class DimColumnResolvedFilterInfo implements Serializable, ResolvedFilter
dimensionResolvedFilter = new HashMap<CarbonDimension, List<DimColumnFilterInfo>>(20);
}
- public IndexKey getStarIndexKey() {
- return starIndexKey;
- }
-
- public void setStarIndexKey(IndexKey starIndexKey) {
- this.starIndexKey = starIndexKey;
- }
-
- public IndexKey getEndIndexKey() {
- return endIndexKey;
- }
-
- public void setEndIndexKey(IndexKey endIndexKey) {
- this.endIndexKey = endIndexKey;
- }
-
public void addDimensionResolvedFilterInstance(CarbonDimension dimension,
DimColumnFilterInfo filterResolvedObj) {
List<DimColumnFilterInfo> currentVals = dimensionResolvedFilter.get(dimension);
@@ -132,14 +97,6 @@ public class DimColumnResolvedFilterInfo implements Serializable, ResolvedFilter
this.columnIndex = columnIndex;
}
- public boolean isNeedCompressedData() {
- return needCompressedData;
- }
-
- public void setNeedCompressedData(boolean needCompressedData) {
- this.needCompressedData = needCompressedData;
- }
-
public DimColumnFilterInfo getFilterValues() {
return resolvedFilterValueObj;
}
@@ -164,14 +121,6 @@ public class DimColumnResolvedFilterInfo implements Serializable, ResolvedFilter
this.isDimensionExistsInCurrentSilce = isDimensionExistsInCurrentSilce;
}
- public int getRsSurrogates() {
- return rsSurrogates;
- }
-
- public void setRsSurrogates(int rsSurrogates) {
- this.rsSurrogates = rsSurrogates;
- }
-
public String getDefaultValue() {
return defaultValue;
}
@@ -180,8 +129,8 @@ public class DimColumnResolvedFilterInfo implements Serializable, ResolvedFilter
this.defaultValue = defaultValue;
}
- @Override public void populateFilterInfoBasedOnColumnType(ResolvedFilterInfoVisitorIntf visitor,
- FilterResolverMetadata metadata) throws FilterUnsupportedException {
+ public void populateFilterInfoBasedOnColumnType(ResolvedFilterInfoVisitorIntf visitor,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
if (null != visitor) {
visitor.populateFilterResolvedInfo(this, metadata);
this.addDimensionResolvedFilterInstance(metadata.getColumnExpression().getDimension(),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
index 8055ae9..c02361b 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
@@ -31,12 +31,6 @@ public class MeasureColumnResolvedFilterInfo implements Serializable {
private int rowIndex = -1;
- private Object uniqueValue;
-
- private String aggregator;
-
- private boolean isMeasureExistsInCurrentSlice = true;
-
private Object defaultValue;
private org.apache.carbondata.core.carbon.metadata.datatype.DataType type;
@@ -57,14 +51,6 @@ public class MeasureColumnResolvedFilterInfo implements Serializable {
this.rowIndex = rowIndex;
}
- public Object getUniqueValue() {
- return uniqueValue;
- }
-
- public void setUniqueValue(Object uniqueValue) {
- this.uniqueValue = uniqueValue;
- }
-
public org.apache.carbondata.core.carbon.metadata.datatype.DataType getType() {
return type;
}
@@ -73,33 +59,12 @@ public class MeasureColumnResolvedFilterInfo implements Serializable {
this.type = dataType;
}
- /**
- * @return Returns the aggregator.
- */
- public String getAggregator() {
- return aggregator;
- }
-
- /**
- * @param aggregator The aggregator to set.
- */
- public void setAggregator(String aggregator) {
- this.aggregator = aggregator;
- }
-
public boolean isMeasureExistsInCurrentSlice() {
- return isMeasureExistsInCurrentSlice;
- }
-
- public void setMeasureExistsInCurrentSlice(boolean isMeasureExistsInCurrentSlice) {
- this.isMeasureExistsInCurrentSlice = isMeasureExistsInCurrentSlice;
+ return true;
}
public Object getDefaultValue() {
return defaultValue;
}
- public void setDefaultValue(double defaultValue) {
- this.defaultValue = defaultValue;
- }
}
[5/9] incubator-carbondata git commit: clean up core
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
index 7072edd..cb66214 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
@@ -97,7 +97,7 @@ public class CarbonDictionarySortIndexWriterImpl implements CarbonDictionarySort
* @param sortIndexList list of sortIndex
* @throws IOException In Case of any I/O errors occurs.
*/
- @Override public void writeSortIndex(List<Integer> sortIndexList) throws IOException {
+ @Override public void writeSortIndex(List<Integer> sortIndexList) {
columnSortInfo.setSort_index(sortIndexList);
}
@@ -108,8 +108,7 @@ public class CarbonDictionarySortIndexWriterImpl implements CarbonDictionarySort
* @param invertedSortIndexList list of sortIndexInverted
* @throws IOException In Case of any I/O errors occurs.
*/
- @Override public void writeInvertedSortIndex(List<Integer> invertedSortIndexList)
- throws IOException {
+ @Override public void writeInvertedSortIndex(List<Integer> invertedSortIndexList) {
columnSortInfo.setSort_index_inverted(invertedSortIndexList);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
index 3a7f0f1..666afe9 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
@@ -27,7 +27,6 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.commons.lang.ArrayUtils;
@@ -44,10 +43,9 @@ public class CarbonDictionarySortInfoPreparator {
* @param dictionary old distinct values
* @param dataType DataType of columns
* @return CarbonDictionarySortInfo returns the column Sort Info
- * @throws CarbonUtilException
*/
public CarbonDictionarySortInfo getDictionarySortInfo(List<String> newDistinctValues,
- Dictionary dictionary, DataType dataType) throws CarbonUtilException {
+ Dictionary dictionary, DataType dataType) {
CarbonDictionarySortModel[] dictionarySortModels =
prepareDictionarySortModels(newDistinctValues, dictionary, dataType);
return createColumnSortInfo(dictionarySortModels);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
index c90d41c..ddf460c 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
@@ -133,8 +133,7 @@ public class CarbonDictionarySortModel implements Comparable<CarbonDictionarySor
* @see Object#hashCode()
*/
@Override public int hashCode() {
- int result = ((memberValue == null) ? 0 : memberValue.hashCode());
- return result;
+ return ((memberValue == null) ? 0 : memberValue.hashCode());
}
/**
@@ -168,13 +167,4 @@ public class CarbonDictionarySortModel implements Comparable<CarbonDictionarySor
return key;
}
- /**
- * Returns member buye
- *
- * @return
- */
- public String getMemberValue() {
- return memberValue;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
index 2462caa..52815b7 100644
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -24,8 +24,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
@@ -43,9 +41,6 @@ import org.apache.commons.lang3.ArrayUtils;
*/
public class DictionaryBasedResultCollector extends AbstractScannedResultCollector {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DictionaryBasedResultCollector.class.getName());
-
public DictionaryBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedVectorResultCollector.java
index 500aa2c..cab7caf 100644
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedVectorResultCollector.java
@@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
@@ -39,9 +37,6 @@ import org.apache.carbondata.scan.result.vector.MeasureDataVectorProcessor;
*/
public class DictionaryBasedVectorResultCollector extends AbstractScannedResultCollector {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DictionaryBasedVectorResultCollector.class.getName());
-
private ColumnVectorInfo[] dictionaryInfo;
private ColumnVectorInfo[] noDictionaryInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
index 74d4170..aa885d3 100644
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
@@ -21,8 +21,6 @@ package org.apache.carbondata.scan.collector.impl;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.scan.model.QueryMeasure;
import org.apache.carbondata.scan.result.AbstractScannedResult;
@@ -33,9 +31,6 @@ import org.apache.carbondata.scan.wrappers.ByteArrayWrapper;
*/
public class RawBasedResultCollector extends AbstractScannedResultCollector {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(RawBasedResultCollector.class.getName());
-
public RawBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/complextypes/ArrayQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/ArrayQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/ArrayQueryType.java
index 8dd6749..4d6d30c 100644
--- a/core/src/main/java/org/apache/carbondata/scan/complextypes/ArrayQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/scan/complextypes/ArrayQueryType.java
@@ -22,7 +22,6 @@ package org.apache.carbondata.scan.complextypes;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.List;
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.scan.filter.GenericQueryType;
@@ -34,7 +33,6 @@ import org.apache.spark.sql.types.*;
public class ArrayQueryType extends ComplexQueryType implements GenericQueryType {
private GenericQueryType children;
- private int keyOrdinalForQuery;
public ArrayQueryType(String name, String parentname, int blockIndex) {
super(name, parentname, blockIndex);
@@ -65,14 +63,6 @@ public class ArrayQueryType extends ComplexQueryType implements GenericQueryType
}
- @Override public void getAllPrimitiveChildren(List<GenericQueryType> primitiveChild) {
- if (children instanceof PrimitiveQueryType) {
- primitiveChild.add(children);
- } else {
- children.getAllPrimitiveChildren(primitiveChild);
- }
- }
-
public void parseBlocksAndReturnComplexColumnByteArray(
DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
DataOutputStream dataOutputStream) throws IOException {
@@ -81,9 +71,7 @@ public class ArrayQueryType extends ComplexQueryType implements GenericQueryType
ByteBuffer byteArray = ByteBuffer.wrap(input);
int dataLength = byteArray.getInt();
dataOutputStream.writeInt(dataLength);
- if (dataLength == 0) {
- // b.putInt(0);
- } else {
+ if (dataLength > 0) {
int columnIndex = byteArray.getInt();
for (int i = 0; i < dataLength; i++) {
children
@@ -93,52 +81,16 @@ public class ArrayQueryType extends ComplexQueryType implements GenericQueryType
}
}
- @Override public int getSurrogateIndex() {
- return 0;
- }
-
- @Override public void setSurrogateIndex(int surrIndex) {
-
- }
-
- @Override public int getBlockIndex() {
- return blockIndex;
- }
-
- @Override public void setBlockIndex(int blockIndex) {
- this.blockIndex = blockIndex;
- }
-
@Override public int getColsCount() {
return children.getColsCount() + 1;
}
- @Override public void parseAndGetResultBytes(ByteBuffer complexData, DataOutputStream dataOutput)
- throws IOException {
- int dataLength = complexData.getInt();
- dataOutput.writeInt(dataLength);
- for (int i = 0; i < dataLength; i++) {
- children.parseAndGetResultBytes(complexData, dataOutput);
- }
- }
-
- @Override public void setKeySize(int[] keyBlockSize) {
- children.setKeySize(keyBlockSize);
- }
-
@Override public DataType getSchemaType() {
return new ArrayType(null, true);
}
- @Override public int getKeyOrdinalForQuery() {
- return keyOrdinalForQuery;
- }
-
- @Override public void setKeyOrdinalForQuery(int keyOrdinalForQuery) {
- this.keyOrdinalForQuery = keyOrdinalForQuery;
- }
-
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) {
+ @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ throws IOException {
readBlockDataChunk(blockChunkHolder);
children.fillRequiredBlockData(blockChunkHolder);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/complextypes/ComplexQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/ComplexQueryType.java
index 0a4c999..d2c6703 100644
--- a/core/src/main/java/org/apache/carbondata/scan/complextypes/ComplexQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/scan/complextypes/ComplexQueryType.java
@@ -19,6 +19,8 @@
*/
package org.apache.carbondata.scan.complextypes;
+import java.io.IOException;
+
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.scan.filter.GenericQueryType;
import org.apache.carbondata.scan.processor.BlocksChunkHolder;
@@ -38,14 +40,6 @@ public class ComplexQueryType {
this.blockIndex = blockIndex;
}
- public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) {
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- children.fillRequiredBlockData(blockChunkHolder);
- }
-
/**
* Method will copy the block chunk holder data to the passed
* byte[], this method is also used by child
@@ -71,7 +65,7 @@ public class ComplexQueryType {
/*
* This method will read the block data chunk from the respective block
*/
- protected void readBlockDataChunk(BlocksChunkHolder blockChunkHolder) {
+ protected void readBlockDataChunk(BlocksChunkHolder blockChunkHolder) throws IOException {
if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
index d5a5acc..979a61b 100644
--- a/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
@@ -22,7 +22,6 @@ package org.apache.carbondata.scan.complextypes;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.List;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
@@ -37,8 +36,6 @@ import org.apache.spark.sql.types.*;
public class PrimitiveQueryType extends ComplexQueryType implements GenericQueryType {
- private int index;
-
private String name;
private String parentname;
@@ -86,26 +83,6 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
}
- @Override public void getAllPrimitiveChildren(List<GenericQueryType> primitiveChild) {
-
- }
-
- @Override public int getSurrogateIndex() {
- return index;
- }
-
- @Override public void setSurrogateIndex(int surrIndex) {
- index = surrIndex;
- }
-
- @Override public int getBlockIndex() {
- return blockIndex;
- }
-
- @Override public void setBlockIndex(int blockIndex) {
- this.blockIndex = blockIndex;
- }
-
@Override public int getColsCount() {
return 1;
}
@@ -119,14 +96,6 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
dataOutputStream.write(currentVal);
}
- @Override public void setKeySize(int[] keyBlockSize) {
- this.keySize = keyBlockSize[this.blockIndex];
- }
-
- @Override public void parseAndGetResultBytes(ByteBuffer complexData, DataOutputStream dataOutput)
- throws IOException {
- }
-
@Override public DataType getSchemaType() {
switch (dataType) {
case INT:
@@ -146,19 +115,12 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
}
}
- @Override public int getKeyOrdinalForQuery() {
- return 0;
- }
-
- @Override public void setKeyOrdinalForQuery(int keyOrdinalForQuery) {
- }
-
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) {
+ @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ throws IOException {
readBlockDataChunk(blockChunkHolder);
}
@Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
-
byte[] data = new byte[keySize];
surrogateData.get(data);
Bits bit = new Bits(new int[]{keySize * 8});
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java
index 12e6436..36d483b 100644
--- a/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java
@@ -40,14 +40,11 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
private List<GenericQueryType> children = new ArrayList<GenericQueryType>();
private String name;
private String parentname;
- private int blockIndex;
- private int keyOrdinalForQuery;
public StructQueryType(String name, String parentname, int blockIndex) {
super(name, parentname, blockIndex);
this.name = name;
this.parentname = parentname;
- this.blockIndex = blockIndex;
}
@Override public void addChildren(GenericQueryType newChild) {
@@ -78,33 +75,6 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
}
- @Override public void getAllPrimitiveChildren(List<GenericQueryType> primitiveChild) {
- for (int i = 0; i < children.size(); i++) {
- GenericQueryType child = children.get(i);
- if (child instanceof PrimitiveQueryType) {
- primitiveChild.add(child);
- } else {
- child.getAllPrimitiveChildren(primitiveChild);
- }
- }
- }
-
- @Override public int getSurrogateIndex() {
- return 0;
- }
-
- @Override public void setSurrogateIndex(int surrIndex) {
-
- }
-
- @Override public int getBlockIndex() {
- return blockIndex;
- }
-
- @Override public void setBlockIndex(int blockIndex) {
- this.blockIndex = blockIndex;
- }
-
@Override public int getColsCount() {
int colsCount = 1;
for (int i = 0; i < children.size(); i++) {
@@ -121,9 +91,7 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
ByteBuffer byteArray = ByteBuffer.wrap(input);
int childElement = byteArray.getInt();
dataOutputStream.writeInt(childElement);
- if (childElement == 0) {
- // b.putInt(0);
- } else {
+ if (childElement > 0){
for (int i = 0; i < childElement; i++) {
children.get(i)
.parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks, rowNumber,
@@ -132,21 +100,6 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
}
}
- @Override public void parseAndGetResultBytes(ByteBuffer complexData, DataOutputStream dataOutput)
- throws IOException {
- int childElement = complexData.getInt();
- dataOutput.writeInt(childElement);
- for (int i = 0; i < childElement; i++) {
- children.get(i).parseAndGetResultBytes(complexData, dataOutput);
- }
- }
-
- @Override public void setKeySize(int[] keyBlockSize) {
- for (int i = 0; i < children.size(); i++) {
- children.get(i).setKeySize(keyBlockSize);
- }
- }
-
@Override public DataType getSchemaType() {
StructField[] fields = new StructField[children.size()];
for (int i = 0; i < children.size(); i++) {
@@ -156,15 +109,8 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
return new StructType(fields);
}
- @Override public int getKeyOrdinalForQuery() {
- return keyOrdinalForQuery;
- }
-
- @Override public void setKeyOrdinalForQuery(int keyOrdinalForQuery) {
- this.keyOrdinalForQuery = keyOrdinalForQuery;
- }
-
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) {
+ @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ throws IOException {
readBlockDataChunk(blockChunkHolder);
for (int i = 0; i < children.size(); i++) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java
index 1f67c84..949771c 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.scan.executor;
+import java.io.IOException;
+
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.model.QueryModel;
@@ -35,8 +37,10 @@ public interface QueryExecutor<E> {
* @param queryModel query details
* @return query result iterator
* @throws QueryExecutionException if any failure while executing the query
+ * @throws IOException if fail to read files
*/
- CarbonIterator<E> execute(QueryModel queryModel) throws QueryExecutionException;
+ CarbonIterator<E> execute(QueryModel queryModel)
+ throws QueryExecutionException, IOException;
/**
* Below method will be used for cleanup
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java b/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java
index 7003184..5634e0f 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java
@@ -40,7 +40,6 @@ public class QueryExecutionException extends Exception {
/**
* Constructor
*
- * @param errorCode The error code for this exception.
* @param msg The error message for this exception.
*/
public QueryExecutionException(String msg) {
@@ -51,7 +50,6 @@ public class QueryExecutionException extends Exception {
/**
* Constructor
*
- * @param errorCode The error code for this exception.
* @param msg The error message for this exception.
*/
public QueryExecutionException(String msg, Throwable t) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
index 0ceb80f..deb9179 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.executor.impl;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -46,12 +47,10 @@ import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeas
import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic;
import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsConstants;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.scan.executor.QueryExecutor;
import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.executor.infos.AggregatorInfo;
@@ -91,7 +90,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
*
* @param queryModel
*/
- protected void initQuery(QueryModel queryModel) throws QueryExecutionException {
+ protected void initQuery(QueryModel queryModel) throws IOException {
StandardLogService.setThreadName(StandardLogService.getPartitionID(
queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName()),
queryModel.getQueryId());
@@ -118,14 +117,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// remove the invalid table blocks, block which is deleted or compacted
cache.removeTableBlocks(queryModel.getInvalidSegmentIds(),
queryModel.getAbsoluteTableIdentifier());
- try {
- List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
- prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(),
- queryModel.getAbsoluteTableIdentifier());
- queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
- } catch (CarbonUtilException e) {
- throw new QueryExecutionException(e);
- }
+ List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
+ prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(),
+ queryModel.getAbsoluteTableIdentifier());
+ queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
queryStatistic
.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
@@ -139,14 +134,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
int aggTypeCount = queryModel.getQueryMeasures().size();
int currentIndex = 0;
- String[] aggTypes = new String[aggTypeCount];
DataType[] dataTypes = new DataType[aggTypeCount];
for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
// adding the data type and aggregation type of all the measure this
// can be used
// to select the aggregator
- aggTypes[currentIndex] = carbonMeasure.getAggregateFunction();
dataTypes[currentIndex] = carbonMeasure.getMeasure().getDataType();
currentIndex++;
}
@@ -157,8 +150,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// 3. query measure
// so calculating the index of the expression start index
// and measure column start index
- queryProperties.aggExpressionStartIndex = queryModel.getQueryMeasures().size();
- queryProperties.measureStartIndex = aggTypes.length - queryModel.getQueryMeasures().size();
queryProperties.filterMeasures = new HashSet<>();
queryProperties.complexFilterDimension = new HashSet<>();
QueryUtil.getAllFilterDimensions(queryModel.getFilterExpressionResolverTree(),
@@ -174,9 +165,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis());
queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
queryModel.setColumnToDictionaryMapping(queryProperties.columnToDictionayMapping);
- // setting the sort dimension index. as it will be updated while getting the sort info
- // so currently setting it to default 0 means sort is not present in any dimension
- queryProperties.sortDimIndexes = new byte[queryModel.getQueryDimension().size()];
}
private List<TableBlockUniqueIdentifier> prepareTableBlockUniqueIdentifier(
@@ -202,9 +190,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
int[] maskByteRanges =
QueryUtil.getMaskedByteRange(queryModel.getQueryDimension(), keyGenerator);
- // getting the masked bytes for query dimension dictionary column
- int[] maskedBytes = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
-
// max key for the dictionary dimension present in the query
byte[] maxKey = null;
try {
@@ -218,13 +203,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
KeyStructureInfo restructureInfos = new KeyStructureInfo();
restructureInfos.setKeyGenerator(keyGenerator);
restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaskedBytes(maskedBytes);
restructureInfos.setMaxKey(maxKey);
return restructureInfos;
}
protected List<BlockExecutionInfo> getBlockExecutionInfos(QueryModel queryModel)
- throws QueryExecutionException {
+ throws IOException, QueryExecutionException {
initQuery(queryModel);
List<BlockExecutionInfo> blockExecutionInfoList = new ArrayList<BlockExecutionInfo>();
// fill all the block execution infos for all the blocks selected in
@@ -237,8 +221,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
queryModel.getTableBlockInfos().get(i).getBlockletInfos()
.getNumberOfBlockletToScan()));
}
- queryProperties.complexDimensionInfoMap =
- blockExecutionInfoList.get(blockExecutionInfoList.size() - 1).getComlexDimensionInfoMap();
if (null != queryModel.getStatisticsRecorder()) {
QueryStatistic queryStatistic = new QueryStatistic();
queryStatistic.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_NUM,
@@ -285,12 +267,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
// adding aggregation info for query
blockExecutionInfo.setAggregatorInfo(getAggregatorInfoForBlock(queryModel, blockIndex));
- // adding query statistics list to record the statistics
- blockExecutionInfo.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
- // setting the limit
- blockExecutionInfo.setLimit(queryModel.getLimit());
- // setting whether detail query or not
- blockExecutionInfo.setDetailQuery(queryModel.isDetailQuery());
// setting whether raw record query or not
blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
// setting the masked byte of the block which will be
@@ -318,8 +294,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
blockExecutionInfo.getComlexDimensionInfoMap()));
List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
FilterUtil.traverseResolverTreeAndGetStartAndEndKey(segmentProperties,
- queryModel.getAbsoluteTableIdentifier(), queryModel.getFilterExpressionResolverTree(),
- listOfStartEndKeys);
+ queryModel.getFilterExpressionResolverTree(), listOfStartEndKeys);
startIndexKey = listOfStartEndKeys.get(0);
endIndexKey = listOfStartEndKeys.get(1);
} else {
@@ -330,8 +305,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
throw new QueryExecutionException(e);
}
}
- blockExecutionInfo.setFileType(
- FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getStorePath()));
//setting the start index key of the block node
blockExecutionInfo.setStartKey(startIndexKey);
//setting the end index key of the block node
@@ -462,10 +435,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
AggregatorInfo aggregatorInfos = RestructureUtil
.getAggregatorInfos(queryModel.getQueryMeasures(),
tableBlock.getSegmentProperties().getMeasures());
- // setting the index of expression in measure aggregators
- aggregatorInfos.setExpressionAggregatorStartIndex(queryProperties.aggExpressionStartIndex);
- // setting the index of measure columns in measure aggregators
- aggregatorInfos.setMeasureAggregatorStartIndex(queryProperties.measureStartIndex);
// setting the measure aggregator for all aggregation function selected
// in query
aggregatorInfos.setMeasureDataTypes(queryProperties.measureDataTypes);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
index 7e0f5a9..92e7fc8 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.executor.impl;
+import java.io.IOException;
import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
@@ -36,7 +37,7 @@ public class DetailQueryExecutor extends AbstractQueryExecutor<BatchResult> {
@Override
public CarbonIterator<BatchResult> execute(QueryModel queryModel)
- throws QueryExecutionException {
+ throws QueryExecutionException, IOException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
return new DetailQueryResultIterator(
blockExecutionInfoList,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java
index bf61be2..c18e8c9 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDime
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorder;
import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.filter.GenericQueryType;
/**
* Holds all the properties required for query execution
@@ -42,25 +41,6 @@ public class QueryExecutorProperties {
* dictionary key
*/
public KeyStructureInfo keyStructureInfo;
- /**
- * as we have multiple type of column aggregation like
- * dimension,expression,measure so this will be used to for getting the
- * measure aggregation start index
- */
- public int measureStartIndex;
- /**
- * query like count(1),count(*) ,etc will used this parameter
- */
- public boolean isFunctionQuery;
- /**
- * aggExpressionStartIndex
- */
- public int aggExpressionStartIndex;
- /**
- * index of the dimension which is present in the order by
- * in a query
- */
- public byte[] sortDimIndexes;
/**
* this will hold the information about the dictionary dimension
@@ -73,10 +53,6 @@ public class QueryExecutorProperties {
*/
public DataType[] measureDataTypes;
/**
- * complex parent index to query mapping
- */
- public Map<Integer, GenericQueryType> complexDimensionInfoMap;
- /**
* all the complex dimension which is on filter
*/
public Set<CarbonDimension> complexFilterDimension;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
index 884a1eb..1f1dd7d 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.executor.impl;
+import java.io.IOException;
import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
@@ -31,8 +32,9 @@ import org.apache.carbondata.scan.result.iterator.VectorDetailQueryResultIterato
*/
public class VectorDetailQueryExecutor extends AbstractQueryExecutor<Object> {
- @Override public CarbonIterator<Object> execute(QueryModel queryModel)
- throws QueryExecutionException {
+ @Override
+ public CarbonIterator<Object> execute(QueryModel queryModel)
+ throws QueryExecutionException, IOException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
return new VectorDetailQueryResultIterator(
blockExecutionInfoList,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java
index 3a2c356..f4f0fe5 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java
@@ -49,22 +49,6 @@ public class AggregatorInfo {
private Object[] defaultValues;
/**
- * In carbon there are three type of aggregation
- * (dimension aggregation, expression aggregation and measure aggregation)
- * Below index will be used to set the start position of expression in measures
- * aggregator array
- */
- private int expressionAggregatorStartIndex;
-
- /**
- * In carbon there are three type of aggregation
- * (dimension aggregation, expression aggregation and measure aggregation)
- * Below index will be used to set the start position of measures in measures
- * aggregator array
- */
- private int measureAggregatorStartIndex;
-
- /**
* Datatype of each measure;
*/
private DataType[] measureDataTypes;
@@ -111,34 +95,6 @@ public class AggregatorInfo {
this.defaultValues = defaultValues;
}
- /**
- * @return the expressionAggregatorStartIndex
- */
- public int getExpressionAggregatorStartIndex() {
- return expressionAggregatorStartIndex;
- }
-
- /**
- * @param expressionAggregatorStartIndex the expressionAggregatorStartIndex to set
- */
- public void setExpressionAggregatorStartIndex(int expressionAggregatorStartIndex) {
- this.expressionAggregatorStartIndex = expressionAggregatorStartIndex;
- }
-
- /**
- * @return the measureAggregatorStartIndex
- */
- public int getMeasureAggregatorStartIndex() {
- return measureAggregatorStartIndex;
- }
-
- /**
- * @param measureAggregatorStartIndex the measureAggregatorStartIndex to set
- */
- public void setMeasureAggregatorStartIndex(int measureAggregatorStartIndex) {
- this.measureAggregatorStartIndex = measureAggregatorStartIndex;
- }
-
public DataType[] getMeasureDataTypes() {
return measureDataTypes;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
index 12ed20f..2d31448 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
@@ -24,8 +24,6 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.carbon.datastore.DataRefNode;
import org.apache.carbondata.core.carbon.datastore.IndexKey;
import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorder;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.scan.filter.GenericQueryType;
import org.apache.carbondata.scan.filter.executer.FilterExecuter;
@@ -52,26 +50,6 @@ public class BlockExecutionInfo {
private boolean isFixedKeyUpdateRequired;
/**
- * in case of detail+order by query when number of output record is same we
- * need to store data in the disk, so for this check will be used to whether
- * we can write in the disk or not
- */
- private boolean isFileBasedQuery;
-
- /**
- * id of the query. this will be used to create directory while writing the
- * data file in case of detail+order by query
- */
- private String queryId;
-
- /**
- * this to handle limit query in case of detail query we are pushing down
- * the limit to executor level so based on the number of limit we can
- * process only that many records
- */
- private int limit;
-
- /**
* below to store all the information required for aggregation during query
* execution
*/
@@ -99,11 +77,6 @@ public class BlockExecutionInfo {
private int[] maskedByteForBlock;
/**
- * flag to check whether query is detail query or aggregation query
- */
- private boolean isDetailQuery;
-
- /**
* total number of dimension in block
*/
private int totalNumberDimensionBlock;
@@ -130,11 +103,6 @@ public class BlockExecutionInfo {
private KeyStructureInfo keyStructureInfo;
/**
- * below will be used to sort the data based
- */
- private SortInfo sortInfo;
-
- /**
* first block from which query execution will start
*/
private DataRefNode firstDataBlock;
@@ -169,11 +137,6 @@ public class BlockExecutionInfo {
private int[] eachColumnValueSize;
/**
- * partition number
- */
- private String partitionId;
-
- /**
* column group block index in file to key structure info mapping
*/
private Map<Integer, KeyStructureInfo> columnGroupToKeyStructureInfo;
@@ -190,11 +153,6 @@ public class BlockExecutionInfo {
private FilterExecuter filterExecuterTree;
/**
- * fileType
- */
- private FileType fileType;
-
- /**
* whether it needs only raw byte records with out aggregation.
*/
private boolean isRawRecordDetailQuery;
@@ -220,11 +178,6 @@ public class BlockExecutionInfo {
private int[] complexColumnParentBlockIndexes;
/**
- * to record the statistics
- */
- private QueryStatisticsRecorder statisticsRecorder;
-
- /**
* @return the tableBlock
*/
public AbstractIndex getDataBlock() {
@@ -268,48 +221,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the isFileBasedQuery
- */
- public boolean isFileBasedQuery() {
- return isFileBasedQuery;
- }
-
- /**
- * @param isFileBasedQuery the isFileBasedQuery to set
- */
- public void setFileBasedQuery(boolean isFileBasedQuery) {
- this.isFileBasedQuery = isFileBasedQuery;
- }
-
- /**
- * @return the queryId
- */
- public String getQueryId() {
- return queryId;
- }
-
- /**
- * @param queryId the queryId to set
- */
- public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- /**
- * @return the limit
- */
- public int getLimit() {
- return limit;
- }
-
- /**
- * @param limit the limit to set
- */
- public void setLimit(int limit) {
- this.limit = limit;
- }
-
- /**
* @return the aggregatorInfos
*/
public AggregatorInfo getAggregatorInfo() {
@@ -368,20 +279,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the isDetailQuery
- */
- public boolean isDetailQuery() {
- return isDetailQuery;
- }
-
- /**
- * @param isDetailQuery the isDetailQuery to set
- */
- public void setDetailQuery(boolean isDetailQuery) {
- this.isDetailQuery = isDetailQuery;
- }
-
- /**
* @return the totalNumberDimensionBlock
*/
public int getTotalNumberDimensionBlock() {
@@ -452,20 +349,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the sortInfos
- */
- public SortInfo getSortInfo() {
- return sortInfo;
- }
-
- /**
- * @param sortInfo the sortInfos to set
- */
- public void setSortInfo(SortInfo sortInfo) {
- this.sortInfo = sortInfo;
- }
-
- /**
* @return the firstDataBlock
*/
public DataRefNode getFirstDataBlock() {
@@ -550,20 +433,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the partitionId
- */
- public String getPartitionId() {
- return partitionId;
- }
-
- /**
- * @param partitionId the partitionId to set
- */
- public void setPartitionId(String partitionId) {
- this.partitionId = partitionId;
- }
-
- /**
* @return the dictionaryColumnBlockIndex
*/
public int[] getDictionaryColumnBlockIndex() {
@@ -607,33 +476,12 @@ public class BlockExecutionInfo {
}
/**
- * @return the columnIdToDcitionaryMapping
- */
- public Map<String, Dictionary> getColumnIdToDcitionaryMapping() {
- return columnIdToDcitionaryMapping;
- }
-
- /**
* @param columnIdToDcitionaryMapping the columnIdToDcitionaryMapping to set
*/
public void setColumnIdToDcitionaryMapping(Map<String, Dictionary> columnIdToDcitionaryMapping) {
this.columnIdToDcitionaryMapping = columnIdToDcitionaryMapping;
}
- /**
- * @return the fileType
- */
- public FileType getFileType() {
- return fileType;
- }
-
- /**
- * @param fileType the fileType to set
- */
- public void setFileType(FileType fileType) {
- this.fileType = fileType;
- }
-
public boolean isRawRecordDetailQuery() {
return isRawRecordDetailQuery;
}
@@ -670,14 +518,6 @@ public class BlockExecutionInfo {
this.complexColumnParentBlockIndexes = complexColumnParentBlockIndexes;
}
- public QueryStatisticsRecorder getStatisticsRecorder() {
- return statisticsRecorder;
- }
-
- public void setStatisticsRecorder(QueryStatisticsRecorder statisticsRecorder) {
- this.statisticsRecorder = statisticsRecorder;
- }
-
public QueryDimension[] getQueryDimensions() {
return queryDimensions;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java
index ad3ff5f..d742e02 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java
@@ -39,11 +39,6 @@ public class KeyStructureInfo {
private int[] maskByteRanges;
/**
- * masked bytes of the query
- */
- private int[] maskedBytes;
-
- /**
* max key for query execution
*/
private byte[] maxKey;
@@ -82,20 +77,6 @@ public class KeyStructureInfo {
}
/**
- * @return the maskedBytes
- */
- public int[] getMaskedBytes() {
- return maskedBytes;
- }
-
- /**
- * @param maskedBytes the maskedBytes to set
- */
- public void setMaskedBytes(int[] maskedBytes) {
- this.maskedBytes = maskedBytes;
- }
-
- /**
* @return the maxKey
*/
public byte[] getMaxKey() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/infos/SortInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/SortInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/SortInfo.java
deleted file mode 100644
index 457e185..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/SortInfo.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.infos;
-
-import java.util.List;
-
-import org.apache.carbondata.scan.model.QueryDimension;
-
-/**
- * Below class holds the order by information about the query
- */
-public class SortInfo {
-
- /**
- * sorting order of a dimension
- */
- private byte[] dimensionSortOrder;
-
- /**
- * byte range of each dimension present in the order by
- */
- private int[][] maskedByteRangeForSorting;
-
- /**
- * dimension indexes which is used in order bye
- */
- private byte[] sortDimensionIndex;
-
- /**
- * mask key of each dimension
- * this will be used to sort the dimension
- */
- private byte[][] dimensionMaskKeyForSorting;
-
- /**
- * sortDimension
- */
- private List<QueryDimension> sortDimension;
-
- /**
- * @return the dimensionSortOrder
- */
- public byte[] getDimensionSortOrder() {
- return dimensionSortOrder;
- }
-
- /**
- * @param dimensionSortOrder the dimensionSortOrder to set
- */
- public void setDimensionSortOrder(byte[] dimensionSortOrder) {
- this.dimensionSortOrder = dimensionSortOrder;
- }
-
- /**
- * @return the maskedByteRangeForSorting
- */
- public int[][] getMaskedByteRangeForSorting() {
- return maskedByteRangeForSorting;
- }
-
- /**
- * @param maskedByteRangeForSorting the maskedByteRangeForSorting to set
- */
- public void setMaskedByteRangeForSorting(int[][] maskedByteRangeForSorting) {
- this.maskedByteRangeForSorting = maskedByteRangeForSorting;
- }
-
- /**
- * @return the sortDimensionIndex
- */
- public byte[] getSortDimensionIndex() {
- return sortDimensionIndex;
- }
-
- /**
- * @param sortDimensionIndex the sortDimensionIndex to set
- */
- public void setSortDimensionIndex(byte[] sortDimensionIndex) {
- this.sortDimensionIndex = sortDimensionIndex;
- }
-
- /**
- * @return the dimensionMaskKeyForSorting
- */
- public byte[][] getDimensionMaskKeyForSorting() {
- return dimensionMaskKeyForSorting;
- }
-
- /**
- * @param dimensionMaskKeyForSorting the dimensionMaskKeyForSorting to set
- */
- public void setDimensionMaskKeyForSorting(byte[][] dimensionMaskKeyForSorting) {
- this.dimensionMaskKeyForSorting = dimensionMaskKeyForSorting;
- }
-
- /**
- * @return the sortDimension
- */
- public List<QueryDimension> getSortDimension() {
- return sortDimension;
- }
-
- /**
- * @param sortDimension the sortDimension to set
- */
- public void setSortDimension(List<QueryDimension> sortDimension) {
- this.sortDimension = sortDimension;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
index 9fdb9ae..3fd81f8 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.executor.util;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -49,11 +50,9 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
import org.apache.carbondata.scan.complextypes.ArrayQueryType;
import org.apache.carbondata.scan.complextypes.PrimitiveQueryType;
import org.apache.carbondata.scan.complextypes.StructQueryType;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.scan.expression.ColumnExpression;
import org.apache.carbondata.scan.expression.Expression;
@@ -261,11 +260,11 @@ public class QueryUtil {
* convert the result from surrogate key to actual data
* @param absoluteTableIdentifier absolute table identifier
* @return dimension unique id to its dictionary map
- * @throws QueryExecutionException
+ * @throws IOException
*/
public static Map<String, Dictionary> getDimensionDictionaryDetail(
List<QueryDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions,
- AbsoluteTableIdentifier absoluteTableIdentifier) throws QueryExecutionException {
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
// to store dimension unique column id list, this is required as
// dimension can be present in
// query dimension, as well as some aggregation function will be applied
@@ -327,10 +326,10 @@ public class QueryUtil {
* @param dictionaryColumnIdList dictionary column list
* @param absoluteTableIdentifier absolute table identifier
* @return dictionary mapping
- * @throws QueryExecutionException
+ * @throws IOException
*/
private static Map<String, Dictionary> getDictionaryMap(List<String> dictionaryColumnIdList,
- AbsoluteTableIdentifier absoluteTableIdentifier) throws QueryExecutionException {
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
// this for dictionary unique identifier
List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers =
getDictionaryColumnUniqueIdentifierList(dictionaryColumnIdList,
@@ -338,12 +337,9 @@ public class QueryUtil {
CacheProvider cacheProvider = CacheProvider.getInstance();
Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider
.createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath());
- List<Dictionary> columnDictionaryList;
- try {
- columnDictionaryList = forwardDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
- } catch (CarbonUtilException e) {
- throw new QueryExecutionException(e);
- }
+
+ List<Dictionary> columnDictionaryList =
+ forwardDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
Map<String, Dictionary> columnDictionaryMap = new HashMap<>(columnDictionaryList.size());
for (int i = 0; i < dictionaryColumnUniqueIdentifiers.size(); i++) {
// TODO: null check for column dictionary, if cache size is less it
@@ -361,8 +357,7 @@ public class QueryUtil {
* @return
*/
private static List<DictionaryColumnUniqueIdentifier> getDictionaryColumnUniqueIdentifierList(
- List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier)
- throws QueryExecutionException {
+ List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier) {
CarbonTable carbonTable =
CarbonMetadata.getInstance().getCarbonTable(carbonTableIdentifier.getTableUniqueName());
List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers =
@@ -370,13 +365,15 @@ public class QueryUtil {
for (String columnId : dictionaryColumnIdList) {
CarbonDimension dimension = CarbonMetadata.getInstance()
.getCarbonDimensionBasedOnColIdentifier(carbonTable, columnId);
- if (null == dimension) {
- throw new QueryExecutionException("The column id " + columnId + " could not be resolved.");
+ if (dimension != null) {
+ dictionaryColumnUniqueIdentifiers.add(
+ new DictionaryColumnUniqueIdentifier(
+ carbonTableIdentifier,
+ dimension.getColumnIdentifier(),
+ dimension.getDataType()
+ )
+ );
}
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
- new DictionaryColumnUniqueIdentifier(carbonTableIdentifier,
- dimension.getColumnIdentifier(), dimension.getDataType());
- dictionaryColumnUniqueIdentifiers.add(dictionaryColumnUniqueIdentifier);
}
return dictionaryColumnUniqueIdentifiers;
}
@@ -411,103 +408,6 @@ public class QueryUtil {
}
/**
- * Below method will be used to get the masked byte range for dimension
- * which is present in order by
- *
- * @param orderByDimensions order by dimension
- * @param generator key generator
- * @param maskedRanges masked byte range for dimension
- * @return range of masked byte for order by dimension
- */
- public static int[][] getMaskedByteRangeForSorting(List<QueryDimension> orderByDimensions,
- KeyGenerator generator, int[] maskedRanges) {
- int[][] dimensionCompareIndex = new int[orderByDimensions.size()][];
- int index = 0;
- for (int i = 0; i < dimensionCompareIndex.length; i++) {
- Set<Integer> integers = new TreeSet<Integer>();
- if (!orderByDimensions.get(i).getDimension().getEncoder().contains(Encoding.DICTIONARY)
- || orderByDimensions.get(i).getDimension().numberOfChild() > 0) {
- continue;
- }
- int[] range =
- generator.getKeyByteOffsets(orderByDimensions.get(i).getDimension().getKeyOrdinal());
- for (int j = range[0]; j <= range[1]; j++) {
- integers.add(j);
- }
- dimensionCompareIndex[index] = new int[integers.size()];
- int j = 0;
- for (Iterator<Integer> iterator = integers.iterator(); iterator.hasNext(); ) {
- Integer integer = (Integer) iterator.next();
- dimensionCompareIndex[index][j++] = integer.intValue();
- }
- index++;
- }
- for (int i = 0; i < dimensionCompareIndex.length; i++) {
- if (null == dimensionCompareIndex[i]) {
- continue;
- }
- int[] range = dimensionCompareIndex[i];
- if (null != range) {
- for (int j = 0; j < range.length; j++) {
- for (int k = 0; k < maskedRanges.length; k++) {
- if (range[j] == maskedRanges[k]) {
- range[j] = k;
- break;
- }
- }
- }
- }
-
- }
- return dimensionCompareIndex;
- }
-
- /**
- * Below method will be used to get the masked key for sorting
- *
- * @param orderDimensions query dimension
- * @param generator key generator
- * @param maskedByteRangeForSorting masked byte range for sorting
- * @param maskedRanges masked range
- * @return masked byte range
- * @throws QueryExecutionException
- */
- public static byte[][] getMaksedKeyForSorting(List<QueryDimension> orderDimensions,
- KeyGenerator generator, int[][] maskedByteRangeForSorting, int[] maskedRanges)
- throws QueryExecutionException {
- byte[][] maskedKey = new byte[orderDimensions.size()][];
- byte[] mdKey = null;
- long[] key = null;
- byte[] maskedMdKey = null;
- try {
- if (null != maskedByteRangeForSorting) {
- for (int i = 0; i < maskedByteRangeForSorting.length; i++) {
- if (null == maskedByteRangeForSorting[i]) {
- continue;
- }
- key = new long[generator.getDimCount()];
- maskedKey[i] = new byte[maskedByteRangeForSorting[i].length];
- key[orderDimensions.get(i).getDimension().getKeyOrdinal()] = Long.MAX_VALUE;
- mdKey = generator.generateKey(key);
- maskedMdKey = new byte[maskedRanges.length];
- for (int k = 0; k < maskedMdKey.length; k++) { // CHECKSTYLE:OFF
- // Approval
- // No:Approval-V1R2C10_001
- maskedMdKey[k] = mdKey[maskedRanges[k]];
- }
- for (int j = 0; j < maskedByteRangeForSorting[i].length; j++) {
- maskedKey[i][j] = maskedMdKey[maskedByteRangeForSorting[i][j]];
- }// CHECKSTYLE:ON
-
- }
- }
- } catch (KeyGenException e) {
- throw new QueryExecutionException(e);
- }
- return maskedKey;
- }
-
- /**
* Below method will be used to get mapping whether dimension is present in
* order by or not
*
@@ -568,12 +468,9 @@ public class QueryUtil {
int[] maskByteRanges = getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
// max key for column group
byte[] maxKey = getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- // get masked key for column group
- int[] maksedByte = getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
restructureInfos.setKeyGenerator(keyGenerator);
restructureInfos.setMaskByteRanges(maskByteRanges);
restructureInfos.setMaxKey(maxKey);
- restructureInfos.setMaskedBytes(maksedByte);
restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils
.toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()])));
rowGroupToItsRSInfo
@@ -728,11 +625,6 @@ public class QueryUtil {
queryDimension
.setDimension(carbonTable.getDimensionByName(tableName, queryDimension.getColumnName()));
}
- // resolve sort dimension
- for (QueryDimension sortDimension : queryModel.getSortDimension()) {
- sortDimension
- .setDimension(carbonTable.getDimensionByName(tableName, sortDimension.getColumnName()));
- }
// resolve query measure
for (QueryMeasure queryMeasure : queryModel.getQueryMeasures()) {
// in case of count start column name will be count * so
@@ -761,23 +653,6 @@ public class QueryUtil {
}
/**
- * Below method will be used to get the index of number type aggregator
- *
- * @param aggType
- * @return index in aggregator
- */
- public static int[] getNumberTypeIndex(List<String> aggType) {
- List<Integer> indexList = new ArrayList<Integer>();
- for (int i = 0; i < aggType.size(); i++) {
- if (CarbonCommonConstants.SUM.equals(aggType.get(i)) || CarbonCommonConstants.AVERAGE
- .equals(aggType.get(i))) {
- indexList.add(i);
- }
- }
- return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()]));
- }
-
- /**
* below method will be used to get the actual type aggregator
*
* @param aggType
@@ -812,12 +687,10 @@ public class QueryUtil {
.getColumnGroupMdKeyOrdinal(colGrpId, dimColumnEvaluatorInfo.getColumnIndex()));
int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- int[] maksedByte = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
KeyStructureInfo restructureInfos = new KeyStructureInfo();
restructureInfos.setKeyGenerator(keyGenerator);
restructureInfos.setMaskByteRanges(maskByteRanges);
restructureInfos.setMaxKey(maxKey);
- restructureInfos.setMaskedBytes(maksedByte);
return restructureInfos;
}
@@ -872,7 +745,7 @@ public class QueryUtil {
return complexTypeMap;
}
- private static GenericQueryType fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap,
+ private static void fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap,
CarbonDimension dimension, Map<Integer, GenericQueryType> complexTypeMap,
int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap) {
int parentBlockIndex = dimensionToBlockIndexMap.get(dimension.getOrdinal());
@@ -884,7 +757,6 @@ public class QueryUtil {
parentBlockIndex =
fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex,
dimension, parentQueryType);
- return parentQueryType;
}
private static int fillChildrenDetails(int[] eachComplexColumnValueSize,
@@ -942,9 +814,6 @@ public class QueryUtil {
* This method will check if a given expression contains a column expression
* recursively and add the dimension instance to the set which holds the dimension
* instances of the complex filter expressions.
- *
- * @param filterDimensions
- * @return
*/
private static void addColumnDimensions(Expression expression,
Set<CarbonDimension> filterDimensions, Set<CarbonMeasure> filterMeasure) {
@@ -981,9 +850,6 @@ public class QueryUtil {
/**
* Below method will be used to fill the children dimension column id
- *
- * @param queryDimensions query dimension
- * @param dictionaryDimensionFromQuery dictionary dimension for query
*/
private static void getChildDimensionOrdinal(CarbonDimension queryDimensions,
Set<Integer> filterDimensionsOrdinal) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/expression/BinaryExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/BinaryExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/BinaryExpression.java
index fbaaa81..e50525d 100644
--- a/core/src/main/java/org/apache/carbondata/scan/expression/BinaryExpression.java
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/BinaryExpression.java
@@ -21,17 +21,9 @@ package org.apache.carbondata.scan.expression;
public abstract class BinaryExpression extends Expression {
- /**
- *
- */
private static final long serialVersionUID = 1L;
- /**
- *
- */
-
protected Expression left;
protected Expression right;
- protected boolean isRangeExpression;
public BinaryExpression(Expression left, Expression right) {
this.left = left;
@@ -48,12 +40,4 @@ public abstract class BinaryExpression extends Expression {
return right;
}
- public boolean isRangeExpression() {
- return isRangeExpression;
- }
-
- public void setRangeExpression(boolean isRangeExpression) {
- this.isRangeExpression = isRangeExpression;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/expression/Expression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/Expression.java b/core/src/main/java/org/apache/carbondata/scan/expression/Expression.java
index a1f2bcf..06f6c72 100644
--- a/core/src/main/java/org/apache/carbondata/scan/expression/Expression.java
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/Expression.java
@@ -46,5 +46,4 @@ public abstract class Expression implements Serializable {
public abstract String getString();
- // public abstract void accept(ExpressionVisitor visitor);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ConditionalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ConditionalExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ConditionalExpression.java
index a87231f..19653b6 100644
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ConditionalExpression.java
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ConditionalExpression.java
@@ -34,10 +34,4 @@ public interface ConditionalExpression {
List<ExpressionResult> getLiterals();
- /**
- * will return the flag of direct dictionary column
- *
- * @return
- */
- boolean isDirectDictionaryColumns();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterIllegalMemberException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterIllegalMemberException.java b/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterIllegalMemberException.java
index f75aaed..96b3930 100644
--- a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterIllegalMemberException.java
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterIllegalMemberException.java
@@ -41,7 +41,6 @@ public class FilterIllegalMemberException extends Exception {
/**
* Constructor
*
- * @param errorCode The error code for this exception.
* @param msg The error message for this exception.
*/
public FilterIllegalMemberException(String msg) {
@@ -52,7 +51,6 @@ public class FilterIllegalMemberException extends Exception {
/**
* Constructor
*
- * @param errorCode The error code for this exception.
* @param msg The error message for this exception.
*/
public FilterIllegalMemberException(String msg, Throwable t) {
@@ -63,8 +61,6 @@ public class FilterIllegalMemberException extends Exception {
/**
* Constructor
*
- * @param errorCode The error code for this exception.
- * @param msg The error message for this exception.
*/
public FilterIllegalMemberException(Throwable t) {
super(t);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java b/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
index fd66555..d54ce69 100644
--- a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
@@ -36,7 +36,6 @@ public class FilterUnsupportedException extends Exception {
/**
* Constructor
*
- * @param errorCode The error code for this exception.
* @param msg The error message for this exception.
*/
public FilterUnsupportedException(String msg) {
@@ -47,7 +46,6 @@ public class FilterUnsupportedException extends Exception {
/**
* Constructor
*
- * @param errorCode The error code for this exception.
* @param msg The error message for this exception.
*/
public FilterUnsupportedException(String msg, Throwable t) {
@@ -58,8 +56,6 @@ public class FilterUnsupportedException extends Exception {
/**
* Constructor
*
- * @param errorCode The error code for this exception.
- * @param msg The error message for this exception.
*/
public FilterUnsupportedException(Throwable t) {
super(t);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
index 1b4a0fc..d4ad063 100644
--- a/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.scan.expression.BinaryExpression;
import org.apache.carbondata.scan.expression.ColumnExpression;
@@ -107,21 +106,4 @@ public abstract class BinaryLogicalExpression extends BinaryExpression {
}
- /**
- * the method will return flag (true or false) depending on the existence of the
- * direct dictionary columns in conditional expression
- *
- * @return the method will return flag (true or false)
- */
- public boolean isDirectDictionaryColumns() {
- List<ColumnExpression> listOfExp =
- new ArrayList<ColumnExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- getColumnList(this, listOfExp);
- for (ColumnExpression ce : listOfExp) {
- if (!ce.getCarbonColumn().hasEncoding(Encoding.DICTIONARY)) {
- return true;
- }
- }
- return false;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
index 44a3a31..621b7ec 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.scan.filter;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
@@ -33,7 +34,6 @@ import org.apache.carbondata.core.carbon.datastore.impl.btree.BTreeDataRefNodeFi
import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.BinaryExpression;
import org.apache.carbondata.scan.expression.Expression;
import org.apache.carbondata.scan.expression.conditional.BinaryConditionalExpression;
@@ -59,11 +59,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
* @param expressionTree , filter expression tree
* @param tableIdentifier ,contains carbon store informations
* @return a filter resolver tree
- * @throws QueryExecutionException
- * @throws FilterUnsupportedException
*/
public FilterResolverIntf getFilterResolver(Expression expressionTree,
- AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException {
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException {
if (null != expressionTree && null != tableIdentifier) {
return getFilterResolvertree(expressionTree, tableIdentifier);
}
@@ -83,11 +81,10 @@ public class FilterExpressionProcessor implements FilterProcessor {
* Step:4 The selected blocks will be send to executers for applying the filters with the help
* of Filter executers.
*
- * @throws QueryExecutionException
*/
public List<DataRefNode> getFilterredBlocks(DataRefNode btreeNode,
FilterResolverIntf filterResolver, AbstractIndex tableSegment,
- AbsoluteTableIdentifier tableIdentifier) throws QueryExecutionException {
+ AbsoluteTableIdentifier tableIdentifier) {
// Need to get the current dimension tables
List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
// getting the start and end index key based on filter for hitting the
@@ -96,7 +93,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
+ "start and end block as per filter resolver");
List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
FilterUtil.traverseResolverTreeAndGetStartAndEndKey(tableSegment.getSegmentProperties(),
- tableIdentifier, filterResolver, listOfStartEndKeys);
+ filterResolver, listOfStartEndKeys);
// reading the first value from list which has start key
IndexKey searchStartKey = listOfStartEndKeys.get(0);
// reading the last value from list which has end key
@@ -141,7 +138,6 @@ public class FilterExpressionProcessor implements FilterProcessor {
/**
* Selects the blocks based on col max and min value.
*
- * @param filterResolver
* @param listOfDataBlocksToScan
* @param dataRefNode
*/
@@ -163,11 +159,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
* @param expressionTree , resolver tree which will hold the resolver tree based on
* filter expression.
* @return FilterResolverIntf type.
- * @throws QueryExecutionException
- * @throws FilterUnsupportedException
*/
private FilterResolverIntf getFilterResolvertree(Expression expressionTree,
- AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException {
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException {
FilterResolverIntf filterEvaluatorTree =
createFilterResolverTree(expressionTree, tableIdentifier);
traverseAndResolveTree(filterEvaluatorTree, tableIdentifier);
@@ -182,18 +176,14 @@ public class FilterExpressionProcessor implements FilterProcessor {
*
* @param filterResolverTree
* @param tableIdentifier
- * @throws FilterUnsupportedException
- * @throws QueryExecutionException
*/
private void traverseAndResolveTree(FilterResolverIntf filterResolverTree,
- AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException {
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException {
if (null == filterResolverTree) {
return;
}
traverseAndResolveTree(filterResolverTree.getLeft(), tableIdentifier);
-
filterResolverTree.resolve(tableIdentifier);
-
traverseAndResolveTree(filterResolverTree.getRight(), tableIdentifier);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
index 35948e3..f955524 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
@@ -19,12 +19,12 @@ package org.apache.carbondata.scan.filter;
* under the License.
*/
+import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.apache.carbondata.core.carbon.datastore.DataRefNode;
import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.scan.expression.Expression;
import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
@@ -38,11 +38,10 @@ public interface FilterProcessor {
* @param expressionTree , filter expression tree
* @param tableIdentifier ,contains carbon store informations.
* @return
- * @throws QueryExecutionException
* @throws FilterUnsupportedException
*/
FilterResolverIntf getFilterResolver(Expression expressionTree,
- AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException;
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException;
/**
* This API is exposed inorder to get the required block reference node
@@ -51,10 +50,8 @@ public interface FilterProcessor {
*
* @param filterResolver DataBlock list with resolved filters
* @return list of DataRefNode.
- * @throws QueryExecutionException
*/
List<DataRefNode> getFilterredBlocks(DataRefNode dataRefNode, FilterResolverIntf filterResolver,
- AbstractIndex segmentIndexBuilder, AbsoluteTableIdentifier tableIdentifier)
- throws QueryExecutionException;
+ AbstractIndex segmentIndexBuilder, AbsoluteTableIdentifier tableIdentifier);
}
[7/9] incubator-carbondata git commit: clean up core
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 49c9373..94b685c 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -18,6 +18,7 @@
*/
package org.apache.carbondata.core.carbon.datastore.chunk.reader.measure.v2;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
@@ -92,7 +93,8 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
* @param blockIndexes blocks range to be read
* @return measure column chunks
*/
- public MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader, int[][] blockIndexes) {
+ public MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
+ throws IOException {
// read the column chunk based on block index and add
MeasureColumnDataChunk[] dataChunks =
new MeasureColumnDataChunk[measureColumnChunkOffsets.size()];
@@ -130,7 +132,8 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
* @param blockIndex block to be read
* @return measure data chunk
*/
- @Override public MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex) {
+ @Override public MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex)
+ throws IOException {
MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
DataChunk2 measureColumnChunk = null;
byte[] measureDataChunk = null;
@@ -141,6 +144,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
.readByteArray(filePath, measureColumnChunkOffsets.get(blockIndex),
measureColumnChunkLength.get(blockIndex));
measureColumnChunk = CarbonUtil.readDataChunk(measureDataChunk);
+ assert measureColumnChunk != null;
dataPage = fileReader.readByteArray(filePath,
measureColumnChunkOffsets.get(blockIndex) + measureColumnChunkLength.get(blockIndex),
measureColumnChunk.data_page_length);
@@ -151,6 +155,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
measureDataChunk = new byte[measureColumnChunkLength.get(blockIndex)];
System.arraycopy(data, 0, measureDataChunk, 0, measureColumnChunkLength.get(blockIndex));
measureColumnChunk = CarbonUtil.readDataChunk(measureDataChunk);
+ assert measureColumnChunk != null;
dataPage = new byte[measureColumnChunk.data_page_length];
System.arraycopy(data, measureColumnChunkLength.get(blockIndex), dataPage, 0,
measureColumnChunk.data_page_length);
@@ -188,13 +193,13 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
* @return measure column chunk array
*/
private MeasureColumnDataChunk[] readMeasureChunksInGroup(FileHolder fileReader,
- int startBlockIndex, int endBlockIndex) {
+ int startBlockIndex, int endBlockIndex) throws IOException {
long currentMeasureOffset = measureColumnChunkOffsets.get(startBlockIndex);
byte[] data = fileReader.readByteArray(filePath, currentMeasureOffset,
(int) (measureColumnChunkOffsets.get(endBlockIndex + 1) - currentMeasureOffset));
MeasureColumnDataChunk[] dataChunks =
new MeasureColumnDataChunk[endBlockIndex - startBlockIndex + 1];
- MeasureColumnDataChunk dataChunk = new MeasureColumnDataChunk();
+ MeasureColumnDataChunk dataChunk = null;
int index = 0;
int copyPoint = 0;
byte[] measureDataChunk = null;
@@ -205,6 +210,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
measureDataChunk = new byte[measureColumnChunkLength.get(i)];
System.arraycopy(data, copyPoint, measureDataChunk, 0, measureColumnChunkLength.get(i));
measureColumnChunk = CarbonUtil.readDataChunk(measureDataChunk);
+ assert measureColumnChunk != null;
dataPage = new byte[measureColumnChunk.data_page_length];
copyPoint += measureColumnChunkLength.get(i);
System.arraycopy(data, copyPoint, dataPage, 0, measureColumnChunk.data_page_length);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
index a9c7343..7049d65 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
@@ -19,6 +19,7 @@
package org.apache.carbondata.core.carbon.datastore.exception;
+import java.io.IOException;
import java.util.Locale;
/**
@@ -26,7 +27,7 @@ import java.util.Locale;
*
* @author Administrator
*/
-public class IndexBuilderException extends Exception {
+public class IndexBuilderException extends IOException {
/**
* default serial version ID.
*/
@@ -40,7 +41,6 @@ public class IndexBuilderException extends Exception {
/**
* Constructor
*
- * @param errorCode The error code for this exception.
* @param msg The error message for this exception.
*/
public IndexBuilderException(String msg) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
index d414d3e..99ae8fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
@@ -151,8 +151,7 @@ public abstract class AbstractBTreeBuilder implements BtreeBuilder {
buffer.get(dictionaryKey);
byte[] nonDictionaryKey = new byte[nonDictonaryKeySize];
buffer.get(nonDictionaryKey);
- IndexKey entry = new IndexKey(dictionaryKey, nonDictionaryKey);
- return entry;
+ return new IndexKey(dictionaryKey, nonDictionaryKey);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
index bff3286..d889bce 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.core.carbon.datastore.impl.btree;
+import java.io.IOException;
+
import org.apache.carbondata.core.carbon.datastore.DataRefNode;
import org.apache.carbondata.core.carbon.datastore.IndexKey;
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
@@ -109,7 +111,6 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* This method will be used to get the max value of all the columns this can
* be used in case of filter query
*
- * @param max value of all the columns
*/
@Override public byte[][] getColumnsMaxValue() {
return maxKeyOfColumns;
@@ -119,7 +120,6 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* This method will be used to get the max value of all the columns this can
* be used in case of filter query
*
- * @param max value of all the columns
*/
@Override public byte[][] getColumnsMinValue() {
return minKeyOfColumns;
@@ -172,7 +172,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* @return dimension data chunks
*/
@Override public DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader,
- int[][] blockIndexes) {
+ int[][] blockIndexes) throws IOException {
// No required here as leaf which will will be use this class will implement its own get
// dimension chunks
return null;
@@ -186,7 +186,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* @return dimension data chunk
*/
@Override public DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader,
- int blockIndex) {
+ int blockIndex) throws IOException {
// No required here as leaf which will will be use this class will implement
// its own get dimension chunks
return null;
@@ -200,7 +200,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* @return measure column data chunk
*/
@Override public MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) {
+ int[][] blockIndexes) throws IOException {
// No required here as leaf which will will be use this class will implement its own get
// measure chunks
return null;
@@ -213,7 +213,8 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* @param blockIndex block index to be read from file
* @return measure data chunk
*/
- @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex) {
+ @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
+ throws IOException {
// No required here as leaf which will will be use this class will implement its own get
// measure chunks
return null;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
index 31273ff..911b643 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -69,8 +69,6 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
* Below method will be used to get the first tentative data block based on
* search key
*
- * @param dataBlocks complete data blocks present
- * @param serachKey key to be search
* @return data block
*/
@Override public DataRefNode findFirstDataBlock(DataRefNode dataRefBlock, IndexKey searchKey) {
@@ -86,8 +84,6 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
* Below method will be used to get the last data tentative block based on
* search key
*
- * @param dataBlocks complete data blocks present
- * @param serachKey key to be search
* @return data block
*/
@Override public DataRefNode findLastDataBlock(DataRefNode dataRefBlock, IndexKey searchKey) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java
index c558763..5be4ed1 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java
@@ -62,7 +62,6 @@ public class BTreeNonLeafNode implements BTreeNode {
* as it is a non leaf node it will have the reference of all the leaf node
* under it, setting all the children
*
- * @param leaf nodes
*/
@Override public void setChildren(BTreeNode[] children) {
this.children = children;
@@ -87,7 +86,6 @@ public class BTreeNonLeafNode implements BTreeNode {
/**
* add a key of a leaf node
*
- * @param leaf node start keys
*/
@Override public void setKey(IndexKey key) {
listOfKeys.add(key);
@@ -135,7 +133,6 @@ public class BTreeNonLeafNode implements BTreeNode {
* This method will be used to get the max value of all the columns this can
* be used in case of filter query
*
- * @param max value of all the columns
*/
@Override public byte[][] getColumnsMaxValue() {
// operation of getting the max value is not supported as its a non leaf
@@ -150,7 +147,6 @@ public class BTreeNonLeafNode implements BTreeNode {
* This method will be used to get the max value of all the columns this can
* be used in case of filter query
*
- * @param min value of all the columns
*/
@Override public byte[][] getColumnsMinValue() {
// operation of getting the min value is not supported as its a non leaf
@@ -183,7 +179,6 @@ public class BTreeNonLeafNode implements BTreeNode {
* Below method will be used to get the dimension chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndex block index to be read
* @return dimension data chunk
*/
@Override public DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java
index 34468a6..d76ce4c 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockBTreeLeafNode.java
@@ -38,7 +38,6 @@ public class BlockBTreeLeafNode extends AbstractBTreeLeafNode {
*
* @param builderInfos builder infos which have required metadata to create a leaf
* node
- * @param leafIndex leaf node index
* @param metadataIndex metadata index
*/
public BlockBTreeLeafNode(BTreeBuilderInfo builderInfos, int metadataIndex, long nodeNumber) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java
index 79ee008..e98eae1 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -18,6 +18,8 @@
*/
package org.apache.carbondata.core.carbon.datastore.impl.btree;
+import java.io.IOException;
+
import org.apache.carbondata.core.carbon.datastore.BTreeBuilderInfo;
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
@@ -85,7 +87,7 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* @return dimension data chunks
*/
@Override public DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader,
- int[][] blockIndexes) {
+ int[][] blockIndexes) throws IOException {
return dimensionChunksReader.readDimensionChunks(fileReader, blockIndexes);
}
@@ -97,7 +99,7 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* @return dimension data chunk
*/
@Override public DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader,
- int blockIndex) {
+ int blockIndex) throws IOException {
return dimensionChunksReader.readDimensionChunk(fileReader, blockIndex);
}
@@ -109,7 +111,7 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* @return measure column data chunk
*/
@Override public MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) {
+ int[][] blockIndexes) throws IOException {
return measureColumnChunkReader.readMeasureChunks(fileReader, blockIndexes);
}
@@ -120,7 +122,8 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* @param blockIndex block index to be read from file
* @return measure data chunk
*/
- @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex) {
+ @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
+ throws IOException {
return measureColumnChunkReader.readMeasureChunk(fileReader, blockIndex);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java
index 4f1ae37..e977aca 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/ChunkCompressorMeta.java
@@ -32,61 +32,5 @@ public class ChunkCompressorMeta implements Serializable {
*/
private static final long serialVersionUID = -6697087170420991140L;
- /**
- * data chunk compressor
- */
- private CompressionCodec compressor;
-
- /**
- * total byte size of all uncompressed pages in this column chunk (including the headers)
- */
- private long uncompressedSize;
-
- /**
- * total byte size of all compressed pages in this column chunk (including the headers)
- */
- private long compressedSize;
-
- /**
- * @return the compressor
- */
- public CompressionCodec getCompressorCodec() {
- return compressor;
- }
-
- /**
- * @param compressor the compressor to set
- */
- public void setCompressor(CompressionCodec compressor) {
- this.compressor = compressor;
- }
-
- /**
- * @return the uncompressedSize
- */
- public long getUncompressedSize() {
- return uncompressedSize;
- }
-
- /**
- * @param uncompressedSize the uncompressedSize to set
- */
- public void setUncompressedSize(long uncompressedSize) {
- this.uncompressedSize = uncompressedSize;
- }
-
- /**
- * @return the compressedSize
- */
- public long getCompressedSize() {
- return compressedSize;
- }
-
- /**
- * @param compressedSize the compressedSize to set
- */
- public void setCompressedSize(long compressedSize) {
- this.compressedSize = compressedSize;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/CompressionCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/CompressionCodec.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/CompressionCodec.java
deleted file mode 100644
index 6f302ec..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/compressor/CompressionCodec.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.carbon.metadata.blocklet.compressor;
-
-/**
- * Compressions supported by Carbon Data.
- */
-public enum CompressionCodec {
-
- /**
- * snappy compression
- */
- SNAPPY,
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java
index 584e51f..f301fc6 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/datachunk/DataChunk.java
@@ -21,8 +21,6 @@ package org.apache.carbondata.core.carbon.metadata.blocklet.datachunk;
import java.io.Serializable;
import java.util.List;
-import org.apache.carbondata.core.carbon.metadata.blocklet.compressor.ChunkCompressorMeta;
-import org.apache.carbondata.core.carbon.metadata.blocklet.sort.SortState;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
@@ -37,23 +35,11 @@ public class DataChunk implements Serializable {
private static final long serialVersionUID = 1L;
/**
- * the compression meta data of a chunk
- */
- private ChunkCompressorMeta chunkCompressionMeta;
-
- /**
* whether this chunk is a row chunk or column chunk
*/
private boolean isRowMajor;
/**
- * the column IDs in this chunk, will have atleast
- * one column ID for columnar format, many column ID for
- * row major format
- */
- private List<Integer> columnUniqueIdList;
-
- /**
* Offset of data page
*/
private long dataPageOffset;
@@ -89,21 +75,6 @@ public class DataChunk implements Serializable {
private int rlePageLength;
/**
- * is rle is applied in the data chunk
- */
- private boolean isRleApplied;
-
- /**
- * is dictionary is applied in the column, only if it is a dimension column
- */
- private boolean isNoDictonaryColumn;
-
- /**
- * sorting type selected for chunk;
- */
- private SortState sortState;
-
- /**
* The List of encoders overriden at node level
*/
private List<Encoding> encodingList;
@@ -115,20 +86,6 @@ public class DataChunk implements Serializable {
private List<ValueEncoderMeta> valueEncoderMetaList;
/**
- * @return the chunkCompressionMeta
- */
- public ChunkCompressorMeta getChunkCompressionMeta() {
- return chunkCompressionMeta;
- }
-
- /**
- * @param chunkCompressionMeta the chunkCompressionMeta to set
- */
- public void setChunkCompressionMeta(ChunkCompressorMeta chunkCompressionMeta) {
- this.chunkCompressionMeta = chunkCompressionMeta;
- }
-
- /**
* @return the isRowMajor
*/
public boolean isRowMajor() {
@@ -143,20 +100,6 @@ public class DataChunk implements Serializable {
}
/**
- * @return the columnUniqueIdList
- */
- public List<Integer> getColumnUniqueIdList() {
- return columnUniqueIdList;
- }
-
- /**
- * @param columnUniqueIdList the columnUniqueIdList to set
- */
- public void setColumnUniqueIdList(List<Integer> columnUniqueIdList) {
- this.columnUniqueIdList = columnUniqueIdList;
- }
-
- /**
* @return the dataPageOffset
*/
public long getDataPageOffset() {
@@ -255,48 +198,6 @@ public class DataChunk implements Serializable {
}
/**
- * @return the isRleApplied
- */
- public boolean isRleApplied() {
- return isRleApplied;
- }
-
- /**
- * @param isRleApplied the isRleApplied to set
- */
- public void setRleApplied(boolean isRleApplied) {
- this.isRleApplied = isRleApplied;
- }
-
- /**
- * @return the isNoDictonaryColumn
- */
- public boolean isNoDictonaryColumn() {
- return isNoDictonaryColumn;
- }
-
- /**
- * @param isNoDictonaryColumn the isNoDictonaryColumn to set
- */
- public void setNoDictonaryColumn(boolean isNoDictonaryColumn) {
- this.isNoDictonaryColumn = isNoDictonaryColumn;
- }
-
- /**
- * @return the sortState
- */
- public SortState getSortState() {
- return sortState;
- }
-
- /**
- * @param sortState the sortState to set
- */
- public void setSortState(SortState sortState) {
- this.sortState = sortState;
- }
-
- /**
* @return the encoderList
*/
public List<Encoding> getEncodingList() {
@@ -304,9 +205,9 @@ public class DataChunk implements Serializable {
}
/**
- * @param encoderList the encoderList to set
+ * @param encodingList the encoderList to set
*/
- public void setEncoderList(List<Encoding> encodingList) {
+ public void setEncodingList(List<Encoding> encodingList) {
this.encodingList = encodingList;
}
@@ -318,7 +219,7 @@ public class DataChunk implements Serializable {
}
/**
- * @param valueEncoderMeta the valueEncoderMeta to set
+ * @param valueEncoderMetaList the valueEncoderMeta to set
*/
public void setValueEncoderMeta(List<ValueEncoderMeta> valueEncoderMetaList) {
this.valueEncoderMetaList = valueEncoderMetaList;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/sort/SortState.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/sort/SortState.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/sort/SortState.java
deleted file mode 100644
index 122198c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/blocklet/sort/SortState.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.metadata.blocklet.sort;
-
-/**
- * Enum for sort type information
- */
-public enum SortState {
-
- /**
- * column is not sorted
- */
- SORT_NONE,
-
- /**
- * data from source was already in sorted order
- */
- SORT_NATIVE,
-
- /**
- * data from source was not sorted,so data is explicitly sorted
- */
- SORT_EXPLICT;
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 4b2be5f..ce50c87 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -356,8 +356,6 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
wrapperColumnSchema.setScale(externalColumnSchema.getScale());
wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
- wrapperColumnSchema.setAggregateFunction(externalColumnSchema.getAggregate_function());
- wrapperColumnSchema.setColumnProperties(externalColumnSchema.getColumnProperties());
wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible());
wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java
index 416f8e7..5a46502 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java
@@ -27,5 +27,5 @@ public enum Encoding {
RLE,
INVERTED_INDEX,
BIT_PACKED,
- DIRECT_DICTIONARY;
+ DIRECT_DICTIONARY
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
index f756ab1..eacacc1 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
@@ -531,10 +531,6 @@ public class CarbonTable implements Serializable {
return blockSize;
}
- public void setBlockSizeInMB(int blockSize) {
- this.blockSize = blockSize;
- }
-
/**
* to get the normal dimension or the primitive dimension of the complex type
*
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java
index 888b898..83013b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableInfo.java
@@ -116,34 +116,6 @@ public class TableInfo implements Serializable {
this.databaseName = databaseName;
}
- public TableSchema getTableSchemaByName(String tableName) {
- if (factTable.getTableName().equalsIgnoreCase(tableName)) {
- return factTable;
- }
- for (TableSchema aggregatTableSchema : aggregateTableList) {
- if (aggregatTableSchema.getTableName().equals(tableName)) {
- return aggregatTableSchema;
- }
- }
- return null;
- }
-
- public TableSchema getTableSchemaByTableId(String tableId) {
- if (factTable.getTableId().equals(tableId)) {
- return factTable;
- }
- for (TableSchema aggregatTableSchema : aggregateTableList) {
- if (aggregatTableSchema.getTableId().equals(tableId)) {
- return aggregatTableSchema;
- }
- }
- return null;
- }
-
- public int getNumberOfAggregateTables() {
- return aggregateTableList.size();
- }
-
/**
* @return the tableUniqueName
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java
index 9beeff2..8db78a8 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/TableSchema.java
@@ -128,36 +128,6 @@ public class TableSchema implements Serializable {
this.tableName = tableName;
}
- /**
- * to get the column schema present in the table by name
- *
- * @param columnName
- * @return column schema if matches the name
- */
- public ColumnSchema getColumnSchemaByName(String columnName) {
- for (ColumnSchema tableColumn : listOfColumns) {
- if (tableColumn.getColumnName().equals(columnName)) {
- return tableColumn;
- }
- }
- return null;
- }
-
- /**
- * to get the column schema present in the table by unique id
- *
- * @param columnUniqueId
- * @return column schema if matches the id
- */
- public ColumnSchema getColumnSchemaById(String columnUniqueId) {
- for (ColumnSchema tableColumn : listOfColumns) {
- if (tableColumn.getColumnUniqueId().equalsIgnoreCase(columnUniqueId)) {
- return tableColumn;
- }
- }
- return null;
- }
-
@Override public int hashCode() {
final int prime = 31;
int result = 1;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java
index ae0a25a..edf9527 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java
@@ -30,11 +30,6 @@ public class CarbonMeasure extends CarbonColumn {
private static final long serialVersionUID = 354341488059013977L;
/**
- * aggregator chosen for measure
- */
- private String aggregateFunction;
-
- /**
* Used when this column contains decimal data.
*/
private int scale;
@@ -69,20 +64,6 @@ public class CarbonMeasure extends CarbonColumn {
}
/**
- * @return the aggregator
- */
- public String getAggregateFunction() {
- return aggregateFunction;
- }
-
- /**
- * @param aggregateFunction the aggregateFunction to set
- */
- public void setAggregateFunction(String aggregateFunction) {
- this.aggregateFunction = aggregateFunction;
- }
-
- /**
* to check whether to dimension are equal or not
*/
@Override public boolean equals(Object obj) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
index c0b2569..ac1caca 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
@@ -102,11 +102,6 @@ public class ColumnSchema implements Serializable {
private int numberOfChild;
/**
- * Used when this column is part of an aggregate function.
- */
- private String aggregateFunction;
-
- /**
* used in case of schema restructuring
*/
private byte[] defaultValue;
@@ -199,7 +194,6 @@ public class ColumnSchema implements Serializable {
}
/**
- * @param columnGroup the columnGroup to set
*/
public void setColumnGroup(int columnGroupId) {
this.columnGroupId = columnGroupId;
@@ -248,20 +242,6 @@ public class ColumnSchema implements Serializable {
}
/**
- * @return the aggregator
- */
- public String getAggregateFunction() {
- return aggregateFunction;
- }
-
- /**
- * @param aggregateFunction the aggregator to set
- */
- public void setAggregateFunction(String aggregateFunction) {
- this.aggregateFunction = aggregateFunction;
- }
-
- /**
* @return the defaultValue
*/
public byte[] getDefaultValue() {
@@ -332,7 +312,6 @@ public class ColumnSchema implements Serializable {
}
/**
- * @param encoderList the encoderList to set
*/
public void setEncodingList(List<Encoding> encodingList) {
this.encodingList = encodingList;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
index 8c9f297..8cf6edb 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
@@ -42,7 +42,6 @@ public class CarbonTablePath extends Path {
protected static final String SCHEMA_FILE = "schema";
protected static final String TABLE_STATUS_FILE = "tablestatus";
protected static final String FACT_DIR = "Fact";
- protected static final String AGGREGATE_TABLE_PREFIX = "Agg";
protected static final String SEGMENT_PREFIX = "Segment_";
protected static final String PARTITION_PREFIX = "Part";
protected static final String CARBON_DATA_EXT = ".carbondata";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java
index 55b878b..4d64242 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorderDummy.java
@@ -30,7 +30,7 @@ public class QueryStatisticsRecorderDummy implements QueryStatisticsRecorder,Ser
*/
private static final long serialVersionUID = -5719752001674467864L;
- public QueryStatisticsRecorderDummy(String queryId) {
+ public QueryStatisticsRecorderDummy() {
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
index 909369d..383a515 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
@@ -67,7 +67,7 @@ public class DoubleCompressor extends ValueCompressor {
for (int j = 0; j < value.length; j++) {
BigDecimal val = BigDecimal.valueOf(value[j]);
double diff = max.subtract(val).doubleValue();
- longResult[i] = (long) (Math.round(diff * Math.pow(10, decimal)));
+ longResult[i] = Math.round(diff * Math.pow(10, decimal));
i++;
}
return longResult;
@@ -122,7 +122,7 @@ public class DoubleCompressor extends ValueCompressor {
case DATA_LONG:
long[] longResult = new long[value.length];
for (int j = 0; j < value.length; j++) {
- longResult[i] = (long) (Math.round(Math.pow(10, decimal) * value[j]));
+ longResult[i] = Math.round(Math.pow(10, decimal) * value[j]);
i++;
}
return longResult;
@@ -188,7 +188,7 @@ public class DoubleCompressor extends ValueCompressor {
default:
double[] defaultResult = new double[value.length];
for (int j = 0; j < value.length; j++) {
- defaultResult[i] = (double) (maxValue - value[j]);
+ defaultResult[i] = maxValue - value[j];
i++;
}
return defaultResult;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 6c159e6..32339ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -778,16 +778,6 @@ public final class CarbonCommonConstants {
public static final String SEGMENT_COMPACTED = "Compacted";
/**
- * property for number of core to load the blocks in driver
- */
- public static final String NUMBER_OF_CORE_TO_LOAD_DRIVER_SEGMENT =
- "no.of.cores.to.load.blocks.in.driver";
- /**
- * default number of cores
- */
- public static final int NUMBER_OF_CORE_TO_LOAD_DRIVER_SEGMENT_DEFAULT_VALUE = 10;
-
- /**
* ZOOKEEPERLOCK TYPE
*/
public static final String CARBON_LOCK_TYPE_ZOOKEEPER =
@@ -833,11 +823,6 @@ public final class CarbonCommonConstants {
public static final String HIVE_CONNECTION_URL = "javax.jdo.option.ConnectionURL";
/**
- * Rocord size in case of compaction.
- */
- public static final int COMPACTION_INMEMORY_RECORD_SIZE = 120000;
-
- /**
* If the level 2 compaction is done in minor then new compacted segment will end with .2
*/
public static String LEVEL2_COMPACTION_INDEX = ".2";
@@ -858,17 +843,6 @@ public final class CarbonCommonConstants {
public static final String ZOOKEEPER_URL = "spark.deploy.zookeeper.url";
/**
- * configure the minimum blocklet size eligible for blocklet distribution
- */
- public static final String CARBON_BLOCKLETDISTRIBUTION_MIN_REQUIRED_SIZE =
- "carbon.blockletdistribution.min.blocklet.size";
-
- /**
- * default blocklet size eligible for blocklet distribution
- */
- public static final int DEFAULT_CARBON_BLOCKLETDISTRIBUTION_MIN_REQUIRED_SIZE = 2;
-
- /**
* File created in case of minor compaction request
*/
public static String minorCompactionRequiredFile = "compactionRequired_minor";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
index 9f8d8ed..1f99158 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
@@ -19,6 +19,7 @@
package org.apache.carbondata.core.datastorage.store;
+import java.io.IOException;
public interface FileHolder {
/**
@@ -30,7 +31,7 @@ public interface FileHolder {
* @param length number of bytes to be read
* @return read byte array
*/
- byte[] readByteArray(String filePath, long offset, int length);
+ byte[] readByteArray(String filePath, long offset, int length) throws IOException;
/**
* This method will be used to read the byte array from file based on length(number of bytes)
@@ -39,7 +40,7 @@ public interface FileHolder {
* @param length number of bytes to be read
* @return read byte array
*/
- byte[] readByteArray(String filePath, int length);
+ byte[] readByteArray(String filePath, int length) throws IOException;
/**
* This method will be used to read int from file from postion(offset), here
@@ -49,7 +50,7 @@ public interface FileHolder {
* @param offset reading start position,
* @return read int
*/
- int readInt(String filePath, long offset);
+ int readInt(String filePath, long offset) throws IOException;
/**
* This method will be used to read long from file from postion(offset), here
@@ -59,7 +60,7 @@ public interface FileHolder {
* @param offset reading start position,
* @return read long
*/
- long readLong(String filePath, long offset);
+ long readLong(String filePath, long offset) throws IOException;
/**
* This method will be used to read int from file from postion(offset), here
@@ -68,7 +69,7 @@ public interface FileHolder {
* @param filePath fully qualified file path
* @return read int
*/
- int readInt(String filePath);
+ int readInt(String filePath) throws IOException;
/**
* This method will be used to read long value from file from postion(offset), here
@@ -78,10 +79,10 @@ public interface FileHolder {
* @param offset reading start position,
* @return read long
*/
- long readDouble(String filePath, long offset);
+ long readDouble(String filePath, long offset) throws IOException;
/**
* This method will be used to close all the streams currently present in the cache
*/
- void finish();
+ void finish() throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java
deleted file mode 100644
index 42c5071..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store;
-
-public interface NodeKeyStore {
- /**
- * This method will be used to get the actual mdkeys array present in the
- * store store
- *
- * @param fileHolder
- * @return mdkey
- */
- byte[] getBackArray(FileHolder fileHolder);
-
- /**
- * This method will be used to insert mdkey to store
- *
- * @param index index of mdkey
- * @param value mdkey
- */
- void put(int index, byte[] value);
-
- /**
- * This method will be used to get the writable key array.
- * writable key array will hold below information:
- * <size of key array><key array>
- * total length will be 4 bytes for size + key array length
- *
- * @return writable array (compressed or normal)
- */
- byte[] getWritableKeyArray();
-
- /**
- * This method will be used to get the mdkkey array based on index
- *
- * @param index index in store
- * @param fileHolder file holder will be used to read the file
- * @return mdkey
- */
- byte[] get(int index, FileHolder fileHolder);
-
- /**
- * This method will clear the store and create the new empty store
- */
- void clear();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
index f2fe9d2..a174a18 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
@@ -32,10 +32,4 @@ public interface NodeMeasureDataStore {
*/
byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
- MeasureDataWrapper getBackData(int[] cols, FileHolder fileHolder);
-
- MeasureDataWrapper getBackData(int cols, FileHolder fileHolder);
-
- short getLength();
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java
deleted file mode 100644
index 9e12847..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-
-public interface ColumnarKeyStore {
- /**
- * This method will be used to get the actual mdkeys array present in the
- * carbon store, it will read and uncomnpress the key
- *
- * @param fileHolder
- * @return mdkey
- * @noDictionaryValKeyIndexes, directkey indexes for determining the NO_DICTIONARY
- * Col inorder to process the direct surrogates data.
- */
- ColumnarKeyStoreDataHolder[] getUnCompressedKeyArray(FileHolder fileHolder, int[] blockIndex,
- boolean[] needCompressedData, int[] noDictionaryValKeyIndexes);
-
- /**
- * This method will be used to get the actual mdkeys array present in the
- * carbon store, it will read and uncomnpress the key
- *
- * @param fileHolder
- * @return mdkey
- */
- ColumnarKeyStoreDataHolder getUnCompressedKeyArray(FileHolder fileHolder, int blockIndex,
- boolean needCompressedData, int[] noDictionaryValKeyIndexes);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
index b66d957..29887a3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
@@ -20,11 +20,9 @@
package org.apache.carbondata.core.datastorage.store.columnar;
import java.nio.ByteBuffer;
-import java.util.List;
public class ColumnarKeyStoreDataHolder {
private byte[] keyblockData;
- private List<byte[]> noDictionaryValBasedKeyBlockData;
private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
@@ -33,34 +31,10 @@ public class ColumnarKeyStoreDataHolder {
this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
}
- //Added constructor for holding noDictionaryValBasedKeyBlockData
- public ColumnarKeyStoreDataHolder(final List<byte[]> noDictionaryValBasedKeyBlockData,
- final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
- this.noDictionaryValBasedKeyBlockData = noDictionaryValBasedKeyBlockData;
+ public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
}
- public byte[] getKeyBlockData() {
- return keyblockData;
- }
-
- /**
- * @return the columnarKeyStoreMetadata
- */
- public ColumnarKeyStoreMetadata getColumnarKeyStoreMetadata() {
- return columnarKeyStoreMetadata;
- }
-
- public void unCompress() {
- if (columnarKeyStoreMetadata.isUnCompressed()) {
- return;
- }
- this.keyblockData = UnBlockIndexer
- .uncompressData(keyblockData, columnarKeyStoreMetadata.getDataIndex(),
- columnarKeyStoreMetadata.getEachRowSize());
- columnarKeyStoreMetadata.setUnCompressed(true);
- }
-
public int getSurrogateKey(int columnIndex) {
byte[] actual = new byte[4];
int startIndex;
@@ -77,21 +51,4 @@ public class ColumnarKeyStoreDataHolder {
return ByteBuffer.wrap(actual).getInt();
}
- /**
- * get the byte[] for high cardinality column block
- *
- * @return List<byte[]>.
- */
- public List<byte[]> getNoDictionaryValBasedKeyBlockData() {
- return noDictionaryValBasedKeyBlockData;
- }
-
- /**
- * set the byte[] for high cardinality column block
- *
- * @param noDictionaryValBasedKeyBlockData
- */
- public void setNoDictionaryValBasedKeyBlockData(List<byte[]> noDictionaryValBasedKeyBlockData) {
- this.noDictionaryValBasedKeyBlockData = noDictionaryValBasedKeyBlockData;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java
deleted file mode 100644
index 8f0621e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
-
-public class ColumnarKeyStoreInfo {
- private int numberOfKeys;
-
- private int[] sizeOfEachBlock;
-
- private int[] keyBlockLengths;
-
- private long[] keyBlockOffsets;
-
- private int[] keyBlockIndexLength;
-
- private long[] keyBlockIndexOffsets;
-
- private String filePath;
-
- private boolean[] isSorted;
-
- private int[] cardinality;
-
- private NumberCompressor numberCompressor;
-
- private NumberCompressor[] keyBlockUnCompressor;
-
- private ColumnGroupModel hybridStoreModel;
-
- /**
- * dataIndexMap
- */
- private int[] dataIndexMapLength;
-
- /**
- * dataIndexMap
- */
- private long[] dataIndexMapOffsets;
-
- /**
- * aggKeyBlock
- */
- private boolean[] aggKeyBlock;
-
- /**
- * @return the numberOfKeys
- */
- public int getNumberOfKeys() {
- return numberOfKeys;
- }
-
- /**
- * @param numberOfKeys the numberOfKeys to set
- */
- public void setNumberOfKeys(int numberOfKeys) {
- this.numberOfKeys = numberOfKeys;
- }
-
- /**
- * @return the sizeOfEachBlock
- */
- public int[] getSizeOfEachBlock() {
- return sizeOfEachBlock;
- }
-
- /**
- * @param sizeOfEachBlock the sizeOfEachBlock to set
- */
- public void setSizeOfEachBlock(int[] sizeOfEachBlock) {
- this.sizeOfEachBlock = sizeOfEachBlock;
- }
-
- /**
- * @return the keyBlockLengths
- */
- public int[] getKeyBlockLengths() {
- return keyBlockLengths;
- }
-
- /**
- * @param keyBlockLengths the keyBlockLengths to set
- */
- public void setKeyBlockLengths(int[] keyBlockLengths) {
- this.keyBlockLengths = keyBlockLengths;
- }
-
- /**
- * @return the keyBlockOffsets
- */
- public long[] getKeyBlockOffsets() {
- return keyBlockOffsets;
- }
-
- /**
- * @param keyBlockOffsets the keyBlockOffsets to set
- */
- public void setKeyBlockOffsets(long[] keyBlockOffsets) {
- this.keyBlockOffsets = keyBlockOffsets;
- }
-
- /**
- * @return the keyBlockIndexLength
- */
- public int[] getKeyBlockIndexLength() {
- return keyBlockIndexLength;
- }
-
- /**
- * @param keyBlockIndexLength the keyBlockIndexLength to set
- */
- public void setKeyBlockIndexLength(int[] keyBlockIndexLength) {
- this.keyBlockIndexLength = keyBlockIndexLength;
- }
-
- /**
- * @return the keyBlockIndexOffsets
- */
- public long[] getKeyBlockIndexOffsets() {
- return keyBlockIndexOffsets;
- }
-
- /**
- * @param keyBlockIndexOffsets the keyBlockIndexOffsets to set
- */
- public void setKeyBlockIndexOffsets(long[] keyBlockIndexOffsets) {
- this.keyBlockIndexOffsets = keyBlockIndexOffsets;
- }
-
- /**
- * @return the filePath
- */
- public String getFilePath() {
- return filePath;
- }
-
- /**
- * @param filePath the filePath to set
- */
- public void setFilePath(String filePath) {
- this.filePath = filePath;
- }
-
- /**
- * @return the isSorted
- */
- public boolean[] getIsSorted() {
- return isSorted;
- }
-
- /**
- * @param isSorted the isSorted to set
- */
- public void setIsSorted(boolean[] isSorted) {
- this.isSorted = isSorted;
- }
-
- /**
- * @return the numberCompressor
- */
- public NumberCompressor getNumberCompressor() {
- return numberCompressor;
- }
-
- /**
- * @param numberCompressor the numberCompressor to set
- */
- public void setNumberCompressor(NumberCompressor numberCompressor) {
- this.numberCompressor = numberCompressor;
- }
-
- /**
- * @return the dataIndexMapLength
- */
- public int[] getDataIndexMapLength() {
- return dataIndexMapLength;
- }
-
- /**
- * @param dataIndexMapLength the dataIndexMapLength to set
- */
- public void setDataIndexMapLength(int[] dataIndexMapLength) {
- this.dataIndexMapLength = dataIndexMapLength;
- }
-
- /**
- * @return the dataIndexMapOffsets
- */
- public long[] getDataIndexMapOffsets() {
- return dataIndexMapOffsets;
- }
-
- /**
- * @param dataIndexMapOffsets the dataIndexMapOffsets to set
- */
- public void setDataIndexMapOffsets(long[] dataIndexMapOffsets) {
- this.dataIndexMapOffsets = dataIndexMapOffsets;
- }
-
- /**
- * @return the aggKeyBlock
- */
- public boolean[] getAggKeyBlock() {
- return aggKeyBlock;
- }
-
- /**
- * @param aggKeyBlock the aggKeyBlock to set
- */
- public void setAggKeyBlock(boolean[] aggKeyBlock) {
- this.aggKeyBlock = aggKeyBlock;
- }
-
- /**
- * @return the keyBlockUnCompressor
- */
- public NumberCompressor[] getKeyBlockUnCompressor() {
- return keyBlockUnCompressor;
- }
-
- /**
- * @param keyBlockUnCompressor the keyBlockUnCompressor to set
- */
- public void setKeyBlockUnCompressor(NumberCompressor[] keyBlockUnCompressor) {
- this.keyBlockUnCompressor = keyBlockUnCompressor;
- }
-
- public int[] getCardinality() {
- return cardinality;
- }
-
- public void setCardinality(int[] cardinality) {
- this.cardinality = cardinality;
- }
-
- public ColumnGroupModel getHybridStoreModel() {
- return hybridStoreModel;
- }
-
- public void setHybridStoreModel(ColumnGroupModel hybridStoreModel) {
- this.hybridStoreModel = hybridStoreModel;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java
index a9dcb35..7754ddb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java
@@ -19,132 +19,35 @@
package org.apache.carbondata.core.datastorage.store.columnar;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
-
-public class ColumnarKeyStoreMetadata {
- private boolean isSorted;
-
- private int[] columnIndex;
+class ColumnarKeyStoreMetadata {
private int[] columnReverseIndex;
private int eachRowSize;
- private int[] dataIndex;
-
- private boolean isUnCompressed;
-
- private KeyGenerator keyGenerator;
-
- /**
- * isNoDictionaryValColumn.
- */
- private boolean isNoDictionaryValColumn;
- private boolean isRowStore;
-
- public ColumnarKeyStoreMetadata(int eachRowSize) {
+ ColumnarKeyStoreMetadata(int eachRowSize) {
this.eachRowSize = eachRowSize;
- keyGenerator = KeyGeneratorFactory.getKeyGenerator(new int[] { eachRowSize });
- }
-
- /**
- * @return the isSorted
- */
- public boolean isSorted() {
- return isSorted;
- }
-
- /**
- * @param isSorted the isSorted to set
- */
- public void setSorted(boolean isSorted) {
- this.isSorted = isSorted;
- }
-
- /**
- * @return the columnIndex
- */
- public int[] getColumnIndex() {
- return columnIndex;
- }
-
- /**
- * @param columnIndex the columnIndex to set
- */
- public void setColumnIndex(int[] columnIndex) {
- this.columnIndex = columnIndex;
}
/**
* @return the eachRowSize
*/
- public int getEachRowSize() {
+ int getEachRowSize() {
return eachRowSize;
}
/**
- * @return the dataIndex
- */
- public int[] getDataIndex() {
- return dataIndex;
- }
-
- /**
- * @param dataIndex the dataIndex to set
- */
- public void setDataIndex(int[] dataIndex) {
- this.dataIndex = dataIndex;
- }
-
- /**
* @return the columnReverseIndex
*/
- public int[] getColumnReverseIndex() {
+ int[] getColumnReverseIndex() {
return columnReverseIndex;
}
/**
* @param columnReverseIndex the columnReverseIndex to set
*/
- public void setColumnReverseIndex(int[] columnReverseIndex) {
+ void setColumnReverseIndex(int[] columnReverseIndex) {
this.columnReverseIndex = columnReverseIndex;
}
- public boolean isUnCompressed() {
- return isUnCompressed;
- }
-
- public void setUnCompressed(boolean isUnCompressed) {
- this.isUnCompressed = isUnCompressed;
- }
-
- public KeyGenerator getKeyGenerator() {
- return keyGenerator;
- }
-
- public boolean isRowStore() {
- return isRowStore;
- }
-
- public void setRowStore(boolean isRowStore) {
- this.isRowStore = isRowStore;
- }
-
- /**
- * @return
- */
- public boolean isNoDictionaryValColumn() {
- return isNoDictionaryValColumn;
-
- }
-
- /**
- * @param isNoDictionaryValColumn
- */
- public void setNoDictionaryValColumn(boolean isNoDictionaryValColumn) {
- this.isNoDictionaryValColumn = isNoDictionaryValColumn;
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
index e68a9fe..824076e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
@@ -221,12 +221,11 @@ public class WriterCompressModel {
this.uniqueValue = uniqueValue;
}
-
public void setCompressionFinders(CompressionFinder[] compressionFinders) {
this.compressionFinders = compressionFinders;
}
- public CompressionFinder[] getCompressionFinder() {
+ public CompressionFinder[] getCompressionFinders() {
return this.compressionFinders;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java
index b8c8248..1b9b163 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java
@@ -129,7 +129,7 @@ public class UnCompressByteArray implements ValueCompressonHolder.UnCompressValu
return holder;
}
- public static enum ByteArrayType {
+ public enum ByteArrayType {
BYTE_ARRAY,
BIG_DECIMAL
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java
index e5c3293..44bb60d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java
@@ -53,7 +53,7 @@ public class UnCompressMaxMinFloat implements UnCompressValue<float[]> {
}
@Override public void setValue(float[] value) {
- this.value = (float[]) value;
+ this.value = value;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java
index e7e2e4e..1c7f651 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneDefault.java
@@ -75,9 +75,6 @@ public class UnCompressNoneDefault implements UnCompressValue<double[]> {
return null;
}
- /**
- * @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
- */
@Override public UnCompressValue getCompressorObject() {
return new UnCompressNoneByte(actualDataType);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java
index e4e5a75..364d2fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/UnCompressNoneLong.java
@@ -117,12 +117,8 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue
private CarbonReadDataHolder unCompressLong() {
CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
-
long[] vals = new long[value.length];
-
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i];
- }
+ System.arraycopy(value, 0, vals, 0, vals.length);
dataHldr.setReadableLongValues(vals);
return dataHldr;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java
index 65c6556..6e2f5d7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java
@@ -23,8 +23,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.store.FileHolder;
@@ -34,9 +32,6 @@ import org.apache.hadoop.fs.Path;
public class DFSFileHolderImpl implements FileHolder {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DFSFileHolderImpl.class.getName());
/**
* cache to hold filename and its stream
*/
@@ -47,10 +42,10 @@ public class DFSFileHolderImpl implements FileHolder {
new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
- @Override public byte[] readByteArray(String filePath, long offset, int length) {
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
FSDataInputStream fileChannel = updateCache(filePath);
- byte[] byteBffer = read(fileChannel, length, offset);
- return byteBffer;
+ return read(fileChannel, length, offset);
}
/**
@@ -61,17 +56,13 @@ public class DFSFileHolderImpl implements FileHolder {
* @param filePath fully qualified file path
* @return channel
*/
- private FSDataInputStream updateCache(String filePath) {
+ private FSDataInputStream updateCache(String filePath) throws IOException {
FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
- try {
- if (null == fileChannel) {
- Path pt = new Path(filePath);
- FileSystem fs = FileSystem.get(FileFactory.getConfiguration());
- fileChannel = fs.open(pt);
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
+ if (null == fileChannel) {
+ Path pt = new Path(filePath);
+ FileSystem fs = FileSystem.get(FileFactory.getConfiguration());
+ fileChannel = fs.open(pt);
+ fileNameAndStreamCache.put(filePath, fileChannel);
}
return fileChannel;
}
@@ -84,14 +75,10 @@ public class DFSFileHolderImpl implements FileHolder {
* @param offset position
* @return byte buffer
*/
- private byte[] read(FSDataInputStream channel, int size, long offset) {
+ private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException {
byte[] byteBffer = new byte[size];
- try {
- channel.seek(offset);
- channel.readFully(byteBffer);
- } catch (Exception e) {
- LOGGER.error(e, e.getMessage());
- }
+ channel.seek(offset);
+ channel.readFully(byteBffer);
return byteBffer;
}
@@ -102,82 +89,46 @@ public class DFSFileHolderImpl implements FileHolder {
* @param size number of bytes
* @return byte buffer
*/
- private byte[] read(FSDataInputStream channel, int size) {
+ private byte[] read(FSDataInputStream channel, int size) throws IOException {
byte[] byteBffer = new byte[size];
- try {
- channel.readFully(byteBffer);
- } catch (Exception e) {
- LOGGER.error(e, e.getMessage());
- }
+ channel.readFully(byteBffer);
return byteBffer;
}
- @Override public int readInt(String filePath, long offset) {
+ @Override public int readInt(String filePath, long offset) throws IOException {
FSDataInputStream fileChannel = updateCache(filePath);
- int i = -1;
- try {
- fileChannel.seek(offset);
- i = fileChannel.readInt();
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
-
- return i;
+ fileChannel.seek(offset);
+ return fileChannel.readInt();
}
- @Override public long readDouble(String filePath, long offset) {
+ @Override public long readDouble(String filePath, long offset) throws IOException {
FSDataInputStream fileChannel = updateCache(filePath);
- long i = -1;
- try {
- fileChannel.seek(offset);
- i = fileChannel.readLong();
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
-
- return i;
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
}
- @Override public void finish() {
+ @Override public void finish() throws IOException {
for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) {
- try {
- FSDataInputStream channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- } catch (IOException exception) {
- LOGGER.error(exception, exception.getMessage());
+ FSDataInputStream channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
}
}
-
}
- @Override public byte[] readByteArray(String filePath, int length) {
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
FSDataInputStream fileChannel = updateCache(filePath);
- byte[] byteBffer = read(fileChannel, length);
- return byteBffer;
+ return read(fileChannel, length);
}
- @Override public long readLong(String filePath, long offset) {
+ @Override public long readLong(String filePath, long offset) throws IOException {
FSDataInputStream fileChannel = updateCache(filePath);
- long i = -1;
- try {
- fileChannel.seek(offset);
- i = fileChannel.readLong();
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return i;
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
}
- @Override public int readInt(String filePath) {
+ @Override public int readInt(String filePath) throws IOException {
FSDataInputStream fileChannel = updateCache(filePath);
- int i = -1;
- try {
- i = fileChannel.readInt();
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return i;
+ return fileChannel.readInt();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java
index c540920..f0c424b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java
@@ -51,20 +51,7 @@ import org.apache.hadoop.io.compress.GzipCodec;
public final class FileFactory {
private static Configuration configuration = null;
- private static FileType storeDefaultFileType = FileType.LOCAL;
-
static {
- String property = CarbonUtil.getCarbonStorePath();
- if (property != null) {
- if (property.startsWith(CarbonUtil.HDFS_PREFIX)) {
- storeDefaultFileType = FileType.HDFS;
- } else if (property.startsWith(CarbonUtil.ALLUXIO_PREFIX)) {
- storeDefaultFileType = FileType.ALLUXIO;
- } else if (property.startsWith(CarbonUtil.VIEWFS_PREFIX)) {
- storeDefaultFileType = FileType.VIEWFS;
- }
- }
-
configuration = new Configuration();
configuration.addResource(new Path("../core-default.xml"));
}
@@ -90,22 +77,6 @@ public final class FileFactory {
}
}
- public static FileType getFileType() {
- String property = CarbonUtil.getCarbonStorePath();
- if (property != null) {
- if (property.startsWith(CarbonUtil.HDFS_PREFIX)) {
- storeDefaultFileType = FileType.HDFS;
- }
- else if (property.startsWith(CarbonUtil.ALLUXIO_PREFIX)) {
- storeDefaultFileType = FileType.ALLUXIO;
- }
- else if (property.startsWith(CarbonUtil.VIEWFS_PREFIX)) {
- storeDefaultFileType = FileType.VIEWFS;
- }
- }
- return storeDefaultFileType;
- }
-
public static FileType getFileType(String path) {
if (path.startsWith(CarbonUtil.HDFS_PREFIX)) {
return FileType.HDFS;
@@ -231,44 +202,6 @@ public final class FileFactory {
}
}
- public static DataOutputStream getDataOutputStream(String path, FileType fileType,
- short replicationFactor) throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.create(pt, replicationFactor);
- return stream;
- default:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
- }
- }
-
- public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize)
- throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path), bufferSize));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.create(pt, true, bufferSize);
- return stream;
- default:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path), bufferSize));
- }
- }
-
public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize,
boolean append) throws IOException {
path = path.replace("\\", "/");
@@ -428,7 +361,6 @@ public final class FileFactory {
return path.delete();
}
-
public static boolean mkdirs(String filePath, FileType fileType) throws IOException {
filePath = filePath.replace("\\", "/");
switch (fileType) {
[6/9] incubator-carbondata git commit: clean up core
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileHolderImpl.java
index 5fefb7b..9a5c06a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileHolderImpl.java
@@ -20,6 +20,7 @@
package org.apache.carbondata.core.datastorage.store.impl;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
@@ -27,18 +28,11 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.store.FileHolder;
public class FileHolderImpl implements FileHolder {
/**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(FileHolderImpl.class.getName());
- /**
* cache to hold filename and its stream
*/
private Map<String, FileChannel> fileNameAndStreamCache;
@@ -65,7 +59,8 @@ public class FileHolderImpl implements FileHolder {
* @param length number of bytes to be read
* @return read byte array
*/
- @Override public byte[] readByteArray(String filePath, long offset, int length) {
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
FileChannel fileChannel = updateCache(filePath);
ByteBuffer byteBffer = read(fileChannel, length, offset);
return byteBffer.array();
@@ -74,19 +69,13 @@ public class FileHolderImpl implements FileHolder {
/**
* This method will be used to close all the streams currently present in the cache
*/
- @Override public void finish() {
-
+ @Override public void finish() throws IOException {
for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
- try {
- FileChannel channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- } catch (IOException exception) {
- LOGGER.error(exception, exception.getMessage());
+ FileChannel channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
}
}
-
}
/**
@@ -97,7 +86,7 @@ public class FileHolderImpl implements FileHolder {
* @param offset reading start position,
* @return read int
*/
- @Override public int readInt(String filePath, long offset) {
+ @Override public int readInt(String filePath, long offset) throws IOException {
FileChannel fileChannel = updateCache(filePath);
ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
return byteBffer.getInt();
@@ -110,7 +99,7 @@ public class FileHolderImpl implements FileHolder {
* @param filePath fully qualified file path
* @return read int
*/
- @Override public int readInt(String filePath) {
+ @Override public int readInt(String filePath) throws IOException {
FileChannel fileChannel = updateCache(filePath);
ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE);
return byteBffer.getInt();
@@ -124,7 +113,7 @@ public class FileHolderImpl implements FileHolder {
* @param offset reading start position,
* @return read int
*/
- @Override public long readDouble(String filePath, long offset) {
+ @Override public long readDouble(String filePath, long offset) throws IOException {
FileChannel fileChannel = updateCache(filePath);
ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
return byteBffer.getLong();
@@ -138,16 +127,12 @@ public class FileHolderImpl implements FileHolder {
* @param filePath fully qualified file path
* @return channel
*/
- private FileChannel updateCache(String filePath) {
+ private FileChannel updateCache(String filePath) throws FileNotFoundException {
FileChannel fileChannel = fileNameAndStreamCache.get(filePath);
- try {
- if (null == fileChannel) {
- FileInputStream stream = new FileInputStream(filePath);
- fileChannel = stream.getChannel();
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
+ if (null == fileChannel) {
+ FileInputStream stream = new FileInputStream(filePath);
+ fileChannel = stream.getChannel();
+ fileNameAndStreamCache.put(filePath, fileChannel);
}
return fileChannel;
}
@@ -160,14 +145,10 @@ public class FileHolderImpl implements FileHolder {
* @param offset position
* @return byte buffer
*/
- private ByteBuffer read(FileChannel channel, int size, long offset) {
+ private ByteBuffer read(FileChannel channel, int size, long offset) throws IOException {
ByteBuffer byteBffer = ByteBuffer.allocate(size);
- try {
- channel.position(offset);
- channel.read(byteBffer);
- } catch (Exception e) {
- LOGGER.error(e, e.getMessage());
- }
+ channel.position(offset);
+ channel.read(byteBffer);
byteBffer.rewind();
return byteBffer;
}
@@ -179,13 +160,9 @@ public class FileHolderImpl implements FileHolder {
* @param size number of bytes
* @return byte buffer
*/
- private ByteBuffer read(FileChannel channel, int size) {
+ private ByteBuffer read(FileChannel channel, int size) throws IOException {
ByteBuffer byteBffer = ByteBuffer.allocate(size);
- try {
- channel.read(byteBffer);
- } catch (Exception e) {
- LOGGER.error(e, e.getMessage());
- }
+ channel.read(byteBffer);
byteBffer.rewind();
return byteBffer;
}
@@ -198,7 +175,7 @@ public class FileHolderImpl implements FileHolder {
* @param length number of bytes to be read
* @return read byte array
*/
- @Override public byte[] readByteArray(String filePath, int length) {
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
FileChannel fileChannel = updateCache(filePath);
ByteBuffer byteBffer = read(fileChannel, length);
return byteBffer.array();
@@ -212,7 +189,7 @@ public class FileHolderImpl implements FileHolder {
* @param offset reading start position,
* @return read long
*/
- @Override public long readLong(String filePath, long offset) {
+ @Override public long readLong(String filePath, long offset) throws IOException {
FileChannel fileChannel = updateCache(filePath);
ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
return byteBffer.getLong();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/MemoryMappedFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/MemoryMappedFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/MemoryMappedFileHolderImpl.java
deleted file mode 100644
index 98d0039..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/MemoryMappedFileHolderImpl.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.store.impl;
-
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-
-public class MemoryMappedFileHolderImpl implements FileHolder {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(MemoryMappedFileHolderImpl.class.getName());
-
- private Map<String, FileChannel> fileNameAndStreamCache;
- private Map<String, MappedByteBuffer> fileNameAndMemoryMappedFileCache;
-
- public MemoryMappedFileHolderImpl() {
- this(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- public MemoryMappedFileHolderImpl(int capacity) {
- this.fileNameAndStreamCache = new HashMap<String, FileChannel>(capacity);
- this.fileNameAndMemoryMappedFileCache = new HashMap<String, MappedByteBuffer>(capacity);
- }
-
- private MappedByteBuffer updateCache(String filePath) {
- MappedByteBuffer byteBuffer = fileNameAndMemoryMappedFileCache.get(filePath);
- try {
- if (null == byteBuffer) {
- FileChannel fileChannel = new RandomAccessFile(filePath, "r").getChannel();
- byteBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size());
- fileNameAndStreamCache.put(filePath, fileChannel);
- fileNameAndMemoryMappedFileCache.put(filePath, byteBuffer);
- }
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return byteBuffer;
- }
-
- @Override
- public byte[] readByteArray(String filePath, long offset, int length) {
- byte[] dst = new byte[length];
- updateCache(filePath).get(dst, (int)offset, length);
- return dst;
- }
-
- @Override
- public byte[] readByteArray(String filePath, int length) {
- byte[] dst = new byte[length];
- updateCache(filePath).get(dst);
- return dst;
- }
-
- @Override
- public int readInt(String filePath, long offset) {
- byte[] dst = readByteArray(filePath, offset, CarbonCommonConstants.INT_SIZE_IN_BYTE);
- return ByteBuffer.wrap(dst).getInt();
- }
-
- @Override
- public long readLong(String filePath, long offset) {
- byte[] dst = readByteArray(filePath, offset, CarbonCommonConstants.LONG_SIZE_IN_BYTE);
- return ByteBuffer.wrap(dst).getLong();
- }
-
- @Override
- public int readInt(String filePath) {
- return updateCache(filePath).getInt();
- }
-
- @Override
- public long readDouble(String filePath, long offset) {
- byte[] dst = readByteArray(filePath, offset, CarbonCommonConstants.LONG_SIZE_IN_BYTE);
- return ByteBuffer.wrap(dst).getLong();
- }
-
- @Override
- public void finish() {
- fileNameAndMemoryMappedFileCache.clear();
- for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
- try {
- FileChannel channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- } catch (IOException exception) {
- LOGGER.error(exception, exception.getMessage());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
index 3adc888..55ed307 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
@@ -64,8 +64,8 @@ public abstract class AbstractHeavyCompressedDoubleArrayDataStore
values[i] = compressionModel.getUnCompressValues()[i].getNew();
if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) {
values[i].setValue(
- ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinder()[i])
- .getCompressedValues(compressionModel.getCompressionFinder()[i], dataHolder[i],
+ ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
+ .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i],
compressionModel.getMaxValue()[i],
compressionModel.getMantissa()[i]));
} else {
@@ -80,8 +80,4 @@ public abstract class AbstractHeavyCompressedDoubleArrayDataStore
return returnValue;
}
- @Override public short getLength() {
- return values != null ? (short) values.length : 0;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
index b2a99c5..5b9e28b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
@@ -19,11 +19,7 @@
package org.apache.carbondata.core.datastorage.store.impl.data.compressed;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.MeasureDataWrapper;
import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.datastorage.store.impl.CompressedDataMeasureDataWrapper;
public class HeavyCompressedDoubleArrayDataInMemoryStore
extends AbstractHeavyCompressedDoubleArrayDataStore {
@@ -32,34 +28,4 @@ public class HeavyCompressedDoubleArrayDataInMemoryStore
super(compressionModel);
}
- @Override public MeasureDataWrapper getBackData(int[] cols, FileHolder fileHolder) {
- if (null == compressionModel) {
- return null;
- }
- CarbonReadDataHolder[] vals = new CarbonReadDataHolder[values.length];
- if (cols != null) {
- for (int i = 0; i < cols.length; i++) {
- vals[cols[i]] = values[cols[i]].uncompress(compressionModel.getConvertedDataType()[cols[i]])
- .getValues(compressionModel.getMantissa()[cols[i]],
- compressionModel.getMaxValue()[cols[i]]);
- }
- } else {
- for (int i = 0; i < vals.length; i++) {
-
- vals[i] = values[i].uncompress(compressionModel.getConvertedDataType()[i])
- .getValues(compressionModel.getMantissa()[i], compressionModel.getMaxValue()[i]);
- }
- }
- return new CompressedDataMeasureDataWrapper(vals);
- }
-
- @Override public MeasureDataWrapper getBackData(int cols, FileHolder fileHolder) {
- if (null == compressionModel) {
- return null;
- }
- CarbonReadDataHolder[] vals = new CarbonReadDataHolder[values.length];
- vals[cols] = values[cols].uncompress(compressionModel.getConvertedDataType()[cols])
- .getValues(compressionModel.getMantissa()[cols], compressionModel.getMaxValue()[cols]);
- return new CompressedDataMeasureDataWrapper(vals);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/DictionaryWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/DictionaryWriter.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/DictionaryWriter.java
index 8c160dc..6e870ee 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/DictionaryWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/DictionaryWriter.java
@@ -18,12 +18,13 @@
*/
package org.apache.carbondata.core.dictionary.generator;
+import java.io.IOException;
/**
* Interface to write dictionary to file system
*/
public interface DictionaryWriter {
- void writeDictionaryData(String tableUniqueName) throws Exception;
+ void writeDictionaryData(String tableUniqueName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
index fe84ef5..f157fa7 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
@@ -106,7 +106,7 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege
}
}
- @Override public void writeDictionaryData(String tableUniqueName) throws Exception {
+ @Override public void writeDictionaryData(String tableUniqueName) throws IOException {
// initialize params
CarbonMetadata metadata = CarbonMetadata.getInstance();
CarbonTable carbonTable = metadata.getCarbonTable(tableUniqueName);
@@ -179,8 +179,6 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege
}
}
}
- } catch (IOException ex) {
- throw ex;
} finally {
if (null != dictionaryWriter) {
dictionaryWriter.close();
@@ -206,7 +204,7 @@ public class IncrementalColumnDictionaryGenerator implements BiDictionary<Intege
DictionaryService dictionaryService,
CarbonTableIdentifier tableIdentifier,
ColumnIdentifier columnIdentifier,
- String storePath) throws Exception{
+ String storePath) throws IOException{
CarbonDictionarySortIndexWriter carbonDictionarySortIndexWriter = null;
try {
CarbonDictionarySortInfoPreparator preparator = new CarbonDictionarySortInfoPreparator();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
index 594ec41..bf88bb6 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
@@ -67,7 +67,7 @@ public class ServerDictionaryGenerator implements DictionaryGenerator<Integer, D
public void writeDictionaryData() throws Exception {
for (String tableUniqueName: tableMap.keySet()) {
TableDictionaryGenerator generator = tableMap.get(tableUniqueName);
- ((DictionaryWriter) generator).writeDictionaryData(tableUniqueName);
+ generator.writeDictionaryData(tableUniqueName);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
index addc3b6..3e8b78e 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
@@ -21,7 +21,12 @@ package org.apache.carbondata.core.dictionary.generator;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -76,7 +81,7 @@ public class TableDictionaryGenerator
return ((BiDictionary) generator).size();
}
- @Override public void writeDictionaryData(String tableUniqueName) throws Exception {
+ @Override public void writeDictionaryData(String tableUniqueName) {
int numOfCores = 1;
final String tableName = tableUniqueName;
try {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
index bb0e8f8..0bf5cda 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
@@ -54,7 +54,7 @@ public class DictionaryServer {
* @param port
* @throws Exception
*/
- public void startServer(int port) throws Exception {
+ public void startServer(int port) {
bootstrap = new ServerBootstrap();
dictionaryServerHandler = new DictionaryServerHandler();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java
index dc50b10..686ea54 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java
@@ -118,7 +118,4 @@ public interface KeyGenerator extends Serializable, Comparator<byte[]> {
*/
int getDimCount();
- int getStartAndEndKeySizeWithOnlyPrimitives();
-
- void setStartAndEndKeySizeWithOnlyPrimitives(int startAndEndKeySizeWithPrimitives);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/ColumnarSplitter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/ColumnarSplitter.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/ColumnarSplitter.java
index b2ea154..c803566 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/ColumnarSplitter.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/ColumnarSplitter.java
@@ -68,24 +68,6 @@ public interface ColumnarSplitter {
byte[] getKeyByteArray(byte[][] key);
/**
- * Takes the split keys and generates the surrogate key array in bytes
- *
- * @param key
- * @param columnIndexes, takes columnIndexes to consider which columns are present in the key
- * @return
- */
- byte[] getKeyByteArray(byte[][] key, int[] columnIndexes);
-
- /**
- * Takes the split keys and generates the surrogate key array
- *
- * @param key
- * @param columnIndexes, takes columnIndexes to consider which columns are present in the key
- * @return
- */
- long[] getKeyArray(byte[][] key, int[] columnIndexes);
-
- /**
* Below method will be used to get the block size
*
* @return
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
index 9276fe3..baf74f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
@@ -22,6 +22,7 @@ package org.apache.carbondata.core.keygenerator.columnar.impl;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
@@ -39,19 +40,10 @@ import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerat
public class MultiDimKeyVarLengthEquiSplitGenerator extends MultiDimKeyVarLengthGenerator
implements ColumnarSplitter {
- /**
- *
- */
private static final long serialVersionUID = -7767757692821917570L;
private byte dimensionsToSplit;
- private int[][] splitDimArray;
-
- private int[][] dimBlockArray;
-
- private int[][][] byteRangesForDims;
-
private int[] blockKeySize;
public MultiDimKeyVarLengthEquiSplitGenerator(int[] lens, byte dimensionsToSplit) {
@@ -89,17 +81,17 @@ public class MultiDimKeyVarLengthEquiSplitGenerator extends MultiDimKeyVarLength
splits[i++] = range;
}
for (int j = 1; j < splits.length; j++) {
- if (splits[j - 1].get(splits[j - 1].size() - 1) == splits[j].get(0)) {
+ if (Objects.equals(splits[j - 1].get(splits[j - 1].size() - 1), splits[j].get(0))) {
splits[j].remove(0);
}
}
- splitDimArray = new int[splits.length][];
+ int[][] splitDimArray = new int[splits.length][];
for (int j = 0; j < splits.length; j++) {
int[] a = convertToArray(splits[j]);
splitDimArray[j] = a.length > 0 ? new int[] { a[0], a[a.length - 1] } : a;
}
- dimBlockArray = new int[byteRangesForKeys.length][];
+ int[][] dimBlockArray = new int[byteRangesForKeys.length][];
Set<Integer>[] dimBlockSet = new Set[dimBlockArray.length];
for (int k = 0; k < byteRangesForKeys.length; k++) {
int[] dimRange = byteRangesForKeys[k];
@@ -127,7 +119,7 @@ public class MultiDimKeyVarLengthEquiSplitGenerator extends MultiDimKeyVarLength
new int[0];
}
- byteRangesForDims = new int[byteRangesForKeys.length][][];
+ int[][][] byteRangesForDims = new int[byteRangesForKeys.length][][];
for (int j = 0; j < byteRangesForKeys.length; j++) {
if (dimBlockArray[j].length > 1) {
int[] bArray1 = splitDimArrayLocalIndexes[dimBlockArray[j][0]];
@@ -207,14 +199,6 @@ public class MultiDimKeyVarLengthEquiSplitGenerator extends MultiDimKeyVarLength
return fullKey;
}
- @Override public byte[] getKeyByteArray(byte[][] key, int[] columnIndexes) {
- return null;
- }
-
- @Override public long[] getKeyArray(byte[][] key, int[] columnIndexes) {
- return null;
- }
-
public int[] getBlockKeySize() {
return blockKeySize;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java
index 34515da..96eb468 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthVariableSplitGenerator.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
@@ -34,9 +35,6 @@ import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerat
public class MultiDimKeyVarLengthVariableSplitGenerator extends MultiDimKeyVarLengthGenerator
implements ColumnarSplitter {
- /**
- *
- */
private static final long serialVersionUID = 1L;
private int[] dimensionsToSplit;
@@ -83,7 +81,7 @@ public class MultiDimKeyVarLengthVariableSplitGenerator extends MultiDimKeyVarLe
splits[i++] = range;
}
for (int j = 1; j < splits.length; j++) {
- if (splits[j - 1].get(splits[j - 1].size() - 1) == splits[j].get(0)) {
+ if (Objects.equals(splits[j - 1].get(splits[j - 1].size() - 1), splits[j].get(0))) {
splits[j].remove(0);
}
}
@@ -202,14 +200,6 @@ public class MultiDimKeyVarLengthVariableSplitGenerator extends MultiDimKeyVarLe
return fullKey;
}
- @Override public byte[] getKeyByteArray(byte[][] key, int[] columnIndexes) {
- return null;
- }
-
- @Override public long[] getKeyArray(byte[][] key, int[] columnIndexes) {
- return null;
- }
-
public int[] getBlockKeySize() {
return blockKeySize;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityConstants.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityConstants.java
index e897843..146d96f 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityConstants.java
@@ -28,12 +28,12 @@ public interface TimeStampGranularityConstants {
* java counts the number of milliseconds from start of "January 1, 1970", this property is
* customized the start of position. for example "January 1, 2000"
*/
- public static final String CARBON_CUTOFF_TIMESTAMP = "carbon.cutOffTimestamp";
+ String CARBON_CUTOFF_TIMESTAMP = "carbon.cutOffTimestamp";
/**
* The property to set the timestamp (ie milis) conversion to the SECOND, MINUTE, HOUR
* or DAY level
*/
- public static final String CARBON_TIME_GRANULARITY = "carbon.timegranularity";
+ String CARBON_TIME_GRANULARITY = "carbon.timegranularity";
/**
* Second level key
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityTypeValue.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityTypeValue.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityTypeValue.java
index 3b50a5c..14e342c 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityTypeValue.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampGranularityTypeValue.java
@@ -49,7 +49,7 @@ public enum TimeStampGranularityTypeValue {
*
* @param value
*/
- private TimeStampGranularityTypeValue(long value) {
+ TimeStampGranularityTypeValue(long value) {
this.value = value;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java
index 76128d4..951d56e 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java
@@ -59,7 +59,7 @@ public abstract class AbstractKeyGenerator implements KeyGenerator {
}
@Override public void setProperty(Object key, Object value) {
- /**
+ /*
* No implementation required.
*/
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
index 6f8dd7d..5fba0cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/MultiDimKeyVarLengthGenerator.java
@@ -29,7 +29,6 @@ public class MultiDimKeyVarLengthGenerator extends AbstractKeyGenerator {
*/
protected int[][] byteRangesForKeys;
private Bits bits;
- private int startAndEndKeySizeWithPrimitives;
public MultiDimKeyVarLengthGenerator(int[] lens) {
bits = new Bits(lens);
@@ -105,13 +104,4 @@ public class MultiDimKeyVarLengthGenerator extends AbstractKeyGenerator {
return bits.getKeyArray(key, maskedByteRanges);
}
- @Override public int getStartAndEndKeySizeWithOnlyPrimitives() {
- return startAndEndKeySizeWithPrimitives;
- }
-
- @Override
- public void setStartAndEndKeySizeWithOnlyPrimitives(int startAndEndKeySizeWithPrimitives) {
- this.startAndEndKeySizeWithPrimitives = startAndEndKeySizeWithPrimitives;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
index 67e766c..62d5bdb 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
@@ -77,8 +77,6 @@ public class HeapMemoryAllocator implements MemoryAllocator {
}
pool.add(new WeakReference<>(memory));
}
- } else {
- // Do nothing
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfo.java
deleted file mode 100644
index 5cfa11b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfo.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.metadata;
-
-public class BlockletInfo {
- /**
- * fileName.
- */
- private String fileName;
-
- /**
- * keyOffset.
- */
- private long keyOffset;
-
- /**
- * measureOffset.
- */
- private long[] measureOffset;
-
- /**
- * measureLength.
- */
- private int[] measureLength;
-
- /**
- * keyLength.
- */
- private int keyLength;
-
- /**
- * numberOfKeys.
- */
- private int numberOfKeys;
-
- /**
- * startKey.
- */
- private byte[] startKey;
-
- /**
- * endKey.
- */
- private byte[] endKey;
-
- /**
- * getFileName().
- *
- * @return String.
- */
- public String getFileName() {
- return fileName;
- }
-
- /**
- * setFileName.
- */
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
-
- /**
- * getKeyOffset.
- *
- * @return long.
- */
- public long getKeyOffset() {
- return keyOffset;
- }
-
- /**
- * setKeyOffset.
- *
- * @param keyOffset
- */
- public void setKeyOffset(long keyOffset) {
- this.keyOffset = keyOffset;
- }
-
- /**
- * getMeasureLength
- *
- * @return int[].
- */
- public int[] getMeasureLength() {
- return measureLength;
- }
-
- /**
- * setMeasureLength.
- *
- * @param measureLength
- */
- public void setMeasureLength(int[] measureLength) {
- this.measureLength = measureLength;
- }
-
- /**
- * getKeyLength.
- *
- * @return
- */
- public int getKeyLength() {
- return keyLength;
- }
-
- /**
- * setKeyLength.
- */
- public void setKeyLength(int keyLength) {
- this.keyLength = keyLength;
- }
-
- /**
- * getMeasureOffset.
- *
- * @return long[].
- */
- public long[] getMeasureOffset() {
- return measureOffset;
- }
-
- /**
- * setMeasureOffset.
- *
- * @param measureOffset
- */
- public void setMeasureOffset(long[] measureOffset) {
- this.measureOffset = measureOffset;
- }
-
- /**
- * getNumberOfKeys()
- *
- * @return int.
- */
- public int getNumberOfKeys() {
- return numberOfKeys;
- }
-
- /**
- * setNumberOfKeys.
- *
- * @param numberOfKeys
- */
- public void setNumberOfKeys(int numberOfKeys) {
- this.numberOfKeys = numberOfKeys;
- }
-
- /**
- * getStartKey().
- *
- * @return byte[].
- */
- public byte[] getStartKey() {
- return startKey;
- }
-
- /**
- * setStartKey.
- *
- * @param startKey
- */
- public void setStartKey(byte[] startKey) {
- this.startKey = startKey;
- }
-
- /**
- * getEndKey().
- *
- * @return byte[].
- */
- public byte[] getEndKey() {
- return endKey;
- }
-
- /**
- * setEndKey.
- *
- * @param endKey
- */
- public void setEndKey(byte[] endKey) {
- this.endKey = endKey;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
index c12c556..c42bc93 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
@@ -22,13 +22,8 @@ package org.apache.carbondata.core.metadata;
import java.util.BitSet;
import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
public class BlockletInfoColumnar {
- /**
- * fileName.
- */
- private String fileName;
/**
* measureOffset.
@@ -91,12 +86,6 @@ public class BlockletInfoColumnar {
private long[] dataIndexMapOffsets;
private boolean[] aggKeyBlock;
- /**
- * blockletMetaSize
- */
- private int blockletMetaSize;
-
- private NumberCompressor[] keyBlockUnCompressor;
private WriterCompressModel compressionModel;
@@ -122,22 +111,6 @@ public class BlockletInfoColumnar {
private BitSet[] measureNullValueIndex;
/**
- * getFileName().
- *
- * @return String.
- */
- public String getFileName() {
- return fileName;
- }
-
- /**
- * setFileName.
- */
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
-
- /**
* getMeasureLength
*
* @return int[].
@@ -300,20 +273,6 @@ public class BlockletInfoColumnar {
}
/**
- * @return the blockletMetaSize
- */
- public int getBlockletMetaSize() {
- return blockletMetaSize;
- }
-
- /**
- * @param blockletMetaSize the blockletMetaSize to set
- */
- public void setBlockletMetaSize(int blockletMetaSize) {
- this.blockletMetaSize = blockletMetaSize;
- }
-
- /**
* @return the dataIndexMapLenght
*/
public int[] getDataIndexMapLength() {
@@ -343,14 +302,6 @@ public class BlockletInfoColumnar {
this.aggKeyBlock = aggKeyBlock;
}
- public NumberCompressor[] getKeyBlockUnCompressor() {
- return keyBlockUnCompressor;
- }
-
- public void setKeyBlockUnCompressor(NumberCompressor[] keyBlockUnCompressor) {
- this.keyBlockUnCompressor = keyBlockUnCompressor;
- }
-
public byte[][] getColumnMaxData() {
return this.columnMaxData;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
index bf475f5..75aecf3 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
@@ -137,9 +137,7 @@ public class CarbonDictionaryReaderImpl implements CarbonDictionaryReader {
readDictionaryMetadataFile();
List<ColumnDictionaryChunk> columnDictionaryChunks =
read(carbonDictionaryColumnMetaChunks, startOffset, endOffset);
- Iterator<byte[]> columnDictionaryChunkWrapper =
- new ColumnDictionaryChunkIterator(columnDictionaryChunks);
- return columnDictionaryChunkWrapper;
+ return (Iterator<byte[]>) new ColumnDictionaryChunkIterator(columnDictionaryChunks);
}
/**
@@ -174,9 +172,7 @@ public class CarbonDictionaryReaderImpl implements CarbonDictionaryReader {
// open dictionary file thrift reader
openThriftReader();
// read the required number of chunks from dictionary file
- List<ColumnDictionaryChunk> columnDictionaryChunks =
- readDictionaryFile(startOffset, dictionaryChunkCountsToBeRead);
- return columnDictionaryChunks;
+ return readDictionaryFile(startOffset, dictionaryChunkCountsToBeRead);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/reader/CarbonFooterReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonFooterReader.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonFooterReader.java
index b9c3ae1..ef4318b 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonFooterReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonFooterReader.java
@@ -65,14 +65,13 @@ public class CarbonFooterReader {
* @return
* @throws IOException
*/
- private ThriftReader openThriftReader(String filePath) throws IOException {
+ private ThriftReader openThriftReader(String filePath) {
- ThriftReader thriftReader = new ThriftReader(filePath, new ThriftReader.TBaseCreator() {
+ return new ThriftReader(filePath, new ThriftReader.TBaseCreator() {
@Override public TBase create() {
return new FileFooter();
}
});
- return thriftReader;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java b/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
index 0958349..81b7661 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
@@ -140,7 +140,7 @@ public class ThriftReader {
* needs a way of obtaining instances of the class to be populated and this interface
* defines the mechanism by which a client provides these instances.
*/
- public static interface TBaseCreator {
+ public interface TBaseCreator {
TBase create();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReader.java b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReader.java
index e0bb413..4d5700c 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReader.java
@@ -34,7 +34,7 @@ public interface CarbonDictionarySortIndexReader extends Closeable {
* @return The method return's the list of dictionary sort Index and sort Index reverse
* @throws IOException In case any I/O error occurs
*/
- public List<Integer> readSortIndex() throws IOException;
+ List<Integer> readSortIndex() throws IOException;
/**
* method for reading the carbon dictionary inverted sort index data
@@ -43,5 +43,5 @@ public interface CarbonDictionarySortIndexReader extends Closeable {
* @return The method return's the list of dictionary inverted sort Index
* @throws IOException In case any I/O error occurs
*/
- public List<Integer> readInvertedSortIndex() throws IOException;
+ List<Integer> readInvertedSortIndex() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/service/ColumnUniqueIdService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/ColumnUniqueIdService.java b/core/src/main/java/org/apache/carbondata/core/service/ColumnUniqueIdService.java
index 1c97082..cc974f0 100644
--- a/core/src/main/java/org/apache/carbondata/core/service/ColumnUniqueIdService.java
+++ b/core/src/main/java/org/apache/carbondata/core/service/ColumnUniqueIdService.java
@@ -30,5 +30,5 @@ public interface ColumnUniqueIdService {
* @param columnSchema
* @return generate unique id
*/
- public String generateUniqueId(String databaseName, ColumnSchema columnSchema);
+ String generateUniqueId(String databaseName, ColumnSchema columnSchema);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java
index 9b9ade6..14ec459 100644
--- a/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java
+++ b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java
@@ -39,7 +39,7 @@ public interface DictionaryService {
* @param carbonStorePath
* @return
*/
- public CarbonDictionaryWriter getDictionaryWriter(CarbonTableIdentifier carbonTableIdentifier,
+ CarbonDictionaryWriter getDictionaryWriter(CarbonTableIdentifier carbonTableIdentifier,
ColumnIdentifier columnIdentifier, String carbonStorePath);
/**
@@ -50,7 +50,7 @@ public interface DictionaryService {
* @param carbonStorePath
* @return
*/
- public CarbonDictionarySortIndexWriter getDictionarySortIndexWriter(
+ CarbonDictionarySortIndexWriter getDictionarySortIndexWriter(
CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
String carbonStorePath);
@@ -62,7 +62,7 @@ public interface DictionaryService {
* @param carbonStorePath
* @return
*/
- public CarbonDictionaryMetadataReader getDictionaryMetadataReader(
+ CarbonDictionaryMetadataReader getDictionaryMetadataReader(
CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
String carbonStorePath);
@@ -74,7 +74,7 @@ public interface DictionaryService {
* @param carbonStorePath
* @return
*/
- public CarbonDictionaryReader getDictionaryReader(CarbonTableIdentifier carbonTableIdentifier,
+ CarbonDictionaryReader getDictionaryReader(CarbonTableIdentifier carbonTableIdentifier,
ColumnIdentifier columnIdentifier, String carbonStorePath);
/**
@@ -85,7 +85,7 @@ public interface DictionaryService {
* @param carbonStorePath
* @return
*/
- public CarbonDictionarySortIndexReader getDictionarySortIndexReader(
+ CarbonDictionarySortIndexReader getDictionarySortIndexReader(
CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
String carbonStorePath);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/unsafe/CarbonUnsafe.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/unsafe/CarbonUnsafe.java b/core/src/main/java/org/apache/carbondata/core/unsafe/CarbonUnsafe.java
index 89ac14a..2e96123 100644
--- a/core/src/main/java/org/apache/carbondata/core/unsafe/CarbonUnsafe.java
+++ b/core/src/main/java/org/apache/carbondata/core/unsafe/CarbonUnsafe.java
@@ -33,7 +33,7 @@ public final class CarbonUnsafe {
try {
Field cause = Unsafe.class.getDeclaredField("theUnsafe");
cause.setAccessible(true);
- unsafe = (Unsafe) cause.get((Object) null);
+ unsafe = (Unsafe) cause.get(null);
} catch (Throwable var2) {
unsafe = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 83763e6..11fa4c1 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -29,14 +29,11 @@ import org.apache.carbondata.core.carbon.datastore.block.BlockInfo;
import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.carbon.metadata.blocklet.SegmentInfo;
-import org.apache.carbondata.core.carbon.metadata.blocklet.compressor.ChunkCompressorMeta;
-import org.apache.carbondata.core.carbon.metadata.blocklet.compressor.CompressionCodec;
import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.PresenceMeta;
import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletBTreeIndex;
import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletMinMaxIndex;
-import org.apache.carbondata.core.carbon.metadata.blocklet.sort.SortState;
import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
@@ -74,7 +71,7 @@ public abstract class AbstractDataFileFooterConverter {
* @throws IOException problem while reading the index file
*/
public List<DataFileFooter> getIndexInfo(String filePath, List<TableBlockInfo> tableBlockInfoList)
- throws IOException, CarbonUtilException {
+ throws IOException {
CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();
try {
@@ -200,7 +197,6 @@ public abstract class AbstractDataFileFooterConverter {
wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
wrapperColumnSchema.setScale(externalColumnSchema.getScale());
wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
- wrapperColumnSchema.setAggregateFunction(externalColumnSchema.getAggregate_function());
return wrapperColumnSchema;
}
@@ -231,23 +227,6 @@ public abstract class AbstractDataFileFooterConverter {
}
/**
- * Below method will be used to convert the thrift compression to wrapper
- * compression codec
- *
- * @param compressionCodecThrift
- * @return wrapper compression codec
- */
- protected CompressionCodec getCompressionCodec(
- org.apache.carbondata.format.CompressionCodec compressionCodecThrift) {
- switch (compressionCodecThrift) {
- case SNAPPY:
- return CompressionCodec.SNAPPY;
- default:
- return CompressionCodec.SNAPPY;
- }
- }
-
- /**
* Below method will be used to convert thrift segment object to wrapper
* segment object
*
@@ -284,23 +263,6 @@ public abstract class AbstractDataFileFooterConverter {
}
/**
- * Below method will be used to convert the thrift compression meta to
- * wrapper chunk compression meta
- *
- * @param chunkCompressionMetaThrift
- * @return chunkCompressionMetaWrapper
- */
- protected ChunkCompressorMeta getChunkCompressionMeta(
- org.apache.carbondata.format.ChunkCompressionMeta chunkCompressionMetaThrift) {
- ChunkCompressorMeta compressorMeta = new ChunkCompressorMeta();
- compressorMeta
- .setCompressor(getCompressionCodec(chunkCompressionMetaThrift.getCompression_codec()));
- compressorMeta.setCompressedSize(chunkCompressionMetaThrift.getTotal_compressed_size());
- compressorMeta.setUncompressedSize(chunkCompressionMetaThrift.getTotal_uncompressed_size());
- return compressorMeta;
- }
-
- /**
* Below method will be used to convert the thrift data type to wrapper data
* type
*
@@ -336,22 +298,6 @@ public abstract class AbstractDataFileFooterConverter {
}
/**
- * Below method will be used to convert the thrift object to wrapper object
- *
- * @param sortStateThrift
- * @return wrapper sort state object
- */
- protected SortState getSortState(org.apache.carbondata.format.SortState sortStateThrift) {
- if (sortStateThrift == org.apache.carbondata.format.SortState.SORT_EXPLICIT) {
- return SortState.SORT_EXPLICT;
- } else if (sortStateThrift == org.apache.carbondata.format.SortState.SORT_NATIVE) {
- return SortState.SORT_NATIVE;
- } else {
- return SortState.SORT_NONE;
- }
- }
-
- /**
* Below method will be used to convert the thrift data chunk to wrapper
* data chunk
*
@@ -361,7 +307,6 @@ public abstract class AbstractDataFileFooterConverter {
protected DataChunk getDataChunk(org.apache.carbondata.format.DataChunk datachunkThrift,
boolean isPresenceMetaPresent) {
DataChunk dataChunk = new DataChunk();
- dataChunk.setColumnUniqueIdList(datachunkThrift.getColumn_ids());
dataChunk.setDataPageLength(datachunkThrift.getData_page_length());
dataChunk.setDataPageOffset(datachunkThrift.getData_page_offset());
if (isPresenceMetaPresent) {
@@ -372,13 +317,11 @@ public abstract class AbstractDataFileFooterConverter {
dataChunk.setRowMajor(datachunkThrift.isRowMajor());
dataChunk.setRowIdPageLength(datachunkThrift.getRowid_page_length());
dataChunk.setRowIdPageOffset(datachunkThrift.getRowid_page_offset());
- dataChunk.setSortState(getSortState(datachunkThrift.getSort_state()));
- dataChunk.setChunkCompressionMeta(getChunkCompressionMeta(datachunkThrift.getChunk_meta()));
List<Encoding> encodingList = new ArrayList<Encoding>(datachunkThrift.getEncoders().size());
for (int i = 0; i < datachunkThrift.getEncoders().size(); i++) {
encodingList.add(fromExternalToWrapperEncoding(datachunkThrift.getEncoders().get(i)));
}
- dataChunk.setEncoderList(encodingList);
+ dataChunk.setEncodingList(encodingList);
if (encodingList.contains(Encoding.DELTA)) {
List<ByteBuffer> thriftEncoderMeta = datachunkThrift.getEncoder_meta();
List<ValueEncoderMeta> encodeMetaList =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java
index 9037e0d..1737a27 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java
@@ -19,6 +19,8 @@
package org.apache.carbondata.core.util;
+import java.io.IOException;
+
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -39,7 +41,7 @@ public class CarbonMergerUtil {
try {
localCardinality = CarbonUtil.getCardinalityFromLevelMetadataFile(
path + '/' + CarbonCommonConstants.LEVEL_METADATA_FILE + tableName + ".metadata");
- } catch (CarbonUtilException e) {
+ } catch (IOException e) {
LOGGER.error("Error occurred :: " + e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 3415d92..602c8ae 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -375,7 +375,7 @@ public class CarbonMetadataUtil {
keyBlockIndexLens[i] = dataChunk.getRowid_page_length();
indexMapOffsets[i] = dataChunk.getRle_page_offset();
indexMapLens[i] = dataChunk.getRle_page_length();
- sortState[i] = dataChunk.getSort_state().equals(SortState.SORT_EXPLICIT) ? true : false;
+ sortState[i] = dataChunk.getSort_state().equals(SortState.SORT_EXPLICIT);
i++;
}
blockletInfoColumnar.setKeyLengths(keyLengths);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index b8077dc..9e6d8b0 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -383,22 +383,6 @@ public final class CarbonProperties {
}
/**
- * Validate the restrictions
- *
- * @param actual
- * @param max
- * @param min
- * @param defaultVal
- * @return
- */
- public long validate(long actual, long max, long min, long defaultVal) {
- if (actual <= max && actual >= min) {
- return actual;
- }
- return defaultVal;
- }
-
- /**
* returns major compaction size value from carbon properties or default value if it is not valid
*
* @return
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
index b7316db..ce166fd 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
@@ -17,7 +17,11 @@
package org.apache.carbondata.core.util;
-import org.apache.carbondata.core.carbon.querystatistics.*;
+import org.apache.carbondata.core.carbon.querystatistics.DriverQueryStatisticsRecorderDummy;
+import org.apache.carbondata.core.carbon.querystatistics.DriverQueryStatisticsRecorderImpl;
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorder;
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorderDummy;
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorderImpl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
public class CarbonTimeStatisticsFactory {
@@ -72,7 +76,7 @@ public class CarbonTimeStatisticsFactory {
if (queryStatisticsRecorderType.equalsIgnoreCase("true")) {
return new QueryStatisticsRecorderImpl(queryId);
} else {
- return new QueryStatisticsRecorderDummy(queryId);
+ return new QueryStatisticsRecorderDummy();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 41af792..27dae6d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -26,7 +26,6 @@ import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.DataInputStream;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -40,7 +39,6 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.concurrent.Callable;
import org.apache.carbondata.common.factory.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
@@ -150,18 +148,6 @@ public final class CarbonUtil {
}
/**
- * @param eachFile
- * @return
- */
- private static String getFolderName(CarbonFile eachFile) {
- String str = eachFile.getAbsolutePath();
- str = str.replace("\\", "/");
- int firstFolderIndex = str.lastIndexOf("/");
- String folderName = str.substring(firstFolderIndex);
- return folderName;
- }
-
- /**
* This method will be used to update the dimension cardinality
*
* @param dimCardinality
@@ -242,7 +228,7 @@ public final class CarbonUtil {
for (int i = 0; i < columnGroups.length; i++) {
columnSplit[i] = columnGroups[i].length;
- columnarStore[i] = columnGroups[i].length > 1 ? false : true;
+ columnarStore[i] = columnGroups[i].length <= 1;
}
ColumnGroupModel colGroupModel = new ColumnGroupModel();
colGroupModel.setNoOfColumnStore(noOfColumnStore);
@@ -296,32 +282,26 @@ public final class CarbonUtil {
* @param path file path array
* @throws Exception exception
*/
- public static void deleteFoldersAndFiles(final File... path) throws CarbonUtilException {
- try {
- UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
+ public static void deleteFoldersAndFiles(final File... path)
+ throws IOException, InterruptedException {
+ UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
- @Override public Void run() throws Exception {
- for (int i = 0; i < path.length; i++) {
- deleteRecursive(path[i]);
- }
- return null;
+ @Override public Void run() throws Exception {
+ for (int i = 0; i < path.length; i++) {
+ deleteRecursive(path[i]);
}
- });
- } catch (IOException e) {
- throw new CarbonUtilException("Error while deleting the folders and files");
- } catch (InterruptedException e) {
- throw new CarbonUtilException("Error while deleting the folders and files");
- }
-
+ return null;
+ }
+ });
}
/**
* Recursively delete the files
*
* @param f File to be deleted
- * @throws CarbonUtilException
+ * @throws IOException
*/
- private static void deleteRecursive(File f) throws CarbonUtilException {
+ private static void deleteRecursive(File f) throws IOException {
if (f.isDirectory()) {
if (f.listFiles() != null) {
for (File c : f.listFiles()) {
@@ -330,26 +310,21 @@ public final class CarbonUtil {
}
}
if (f.exists() && !f.delete()) {
- throw new CarbonUtilException("Error while deleting the folders and files");
+ throw new IOException("Error while deleting the folders and files");
}
}
- public static void deleteFoldersAndFiles(final CarbonFile... file) throws CarbonUtilException {
- try {
- UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
+ public static void deleteFoldersAndFiles(final CarbonFile... file)
+ throws IOException, InterruptedException {
+ UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
- @Override public Void run() throws Exception {
- for (int i = 0; i < file.length; i++) {
- deleteRecursive(file[i]);
- }
- return null;
+ @Override public Void run() throws Exception {
+ for (int i = 0; i < file.length; i++) {
+ deleteRecursive(file[i]);
}
- });
- } catch (IOException e) {
- throw new CarbonUtilException("Error while deleting the folders and files");
- } catch (InterruptedException e) {
- throw new CarbonUtilException("Error while deleting the folders and files");
- }
+ return null;
+ }
+ });
}
public static String getBadLogPath(String storeLocation) {
@@ -361,31 +336,25 @@ public final class CarbonUtil {
}
public static void deleteFoldersAndFilesSilent(final CarbonFile... file)
- throws CarbonUtilException {
- try {
- UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
+ throws IOException, InterruptedException {
+ UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<Void>() {
- @Override public Void run() throws Exception {
- for (int i = 0; i < file.length; i++) {
- deleteRecursiveSilent(file[i]);
- }
- return null;
+ @Override public Void run() throws Exception {
+ for (int i = 0; i < file.length; i++) {
+ deleteRecursiveSilent(file[i]);
}
- });
- } catch (IOException e) {
- throw new CarbonUtilException("Error while deleting the folders and files");
- } catch (InterruptedException e) {
- throw new CarbonUtilException("Error while deleting the folders and files");
- }
+ return null;
+ }
+ });
}
/**
* Recursively delete the files
*
* @param f File to be deleted
- * @throws CarbonUtilException
+ * @throws IOException
*/
- private static void deleteRecursive(CarbonFile f) throws CarbonUtilException {
+ private static void deleteRecursive(CarbonFile f) throws IOException {
if (f.isDirectory()) {
if (f.listFiles() != null) {
for (CarbonFile c : f.listFiles()) {
@@ -394,11 +363,11 @@ public final class CarbonUtil {
}
}
if (f.exists() && !f.delete()) {
- throw new CarbonUtilException("Error while deleting the folders and files");
+ throw new IOException("Error while deleting the folders and files");
}
}
- private static void deleteRecursiveSilent(CarbonFile f) throws CarbonUtilException {
+ private static void deleteRecursiveSilent(CarbonFile f) {
if (f.isDirectory()) {
if (f.listFiles() != null) {
for (CarbonFile c : f.listFiles()) {
@@ -411,10 +380,10 @@ public final class CarbonUtil {
}
}
- public static void deleteFiles(File[] intermediateFiles) throws CarbonUtilException {
+ public static void deleteFiles(File[] intermediateFiles) throws IOException {
for (int i = 0; i < intermediateFiles.length; i++) {
if (!intermediateFiles[i].delete()) {
- throw new CarbonUtilException("Problem while deleting intermediate file");
+ throw new IOException("Problem while deleting intermediate file");
}
}
}
@@ -530,10 +499,10 @@ public final class CarbonUtil {
*
* @param levelPath
* @return
- * @throws CarbonUtilException
+ * @throws IOException
*/
public static int[] getCardinalityFromLevelMetadataFile(String levelPath)
- throws CarbonUtilException {
+ throws IOException {
DataInputStream dataInputStream = null;
int[] cardinality = null;
@@ -548,10 +517,6 @@ public final class CarbonUtil {
cardinality[i] = dataInputStream.readInt();
}
}
- } catch (FileNotFoundException e) {
- throw new CarbonUtilException("Problem while getting the file", e);
- } catch (IOException e) {
- throw new CarbonUtilException("Problem while reading the file", e);
} finally {
closeStreams(dataInputStream);
}
@@ -775,7 +740,6 @@ public final class CarbonUtil {
* Below method will be used to get the value compression model of the
* measure data chunk
*
- * @param measureDataChunkList
* @return value compression model
*/
public static WriterCompressModel getValueCompressionModel(
@@ -880,15 +844,11 @@ public final class CarbonUtil {
* Below method will be used to read the data file matadata
*/
public static DataFileFooter readMetadatFile(TableBlockInfo tableBlockInfo)
- throws CarbonUtilException {
+ throws IOException {
AbstractDataFileFooterConverter fileFooterConverter =
DataFileFooterConverterFactory.getInstance()
.getDataFileFooterConverter(tableBlockInfo.getVersion());
- try {
- return fileFooterConverter.readDataFileFooter(tableBlockInfo);
- } catch (IOException e) {
- throw new CarbonUtilException("Problem while reading the file metadata", e);
- }
+ return fileFooterConverter.readDataFileFooter(tableBlockInfo);
}
/**
@@ -898,7 +858,8 @@ public final class CarbonUtil {
* @param blockLength
* @return
*/
- public static long calculateMetaSize(String filePath, long blockOffset, long blockLength) {
+ public static long calculateMetaSize(String filePath, long blockOffset, long blockLength)
+ throws IOException {
FileHolder fileReader = null;
try {
long completeBlockLength = blockOffset + blockLength;
@@ -910,7 +871,11 @@ public final class CarbonUtil {
}
finally {
if(null != fileReader) {
- fileReader.finish();
+ try {
+ fileReader.finish();
+ } catch (IOException e) {
+ // ignore the exception as nothing we can do about it
+ }
}
}
}
@@ -973,23 +938,6 @@ public final class CarbonUtil {
}
}
}
- /**
- * Thread to delete the tables
- *
- */
- private static final class DeleteFolderAndFiles implements Callable<Void> {
- private CarbonFile file;
-
- private DeleteFolderAndFiles(CarbonFile file) {
- this.file = file;
- }
-
- @Override public Void call() throws Exception {
- deleteFoldersAndFiles(file);
- return null;
- }
-
- }
/**
* Below method will be used to get the dimension
@@ -1159,11 +1107,11 @@ public final class CarbonUtil {
* @param tableBlockInfoList list of table block
* @param absoluteTableIdentifier absolute table identifier
* @return list of block info
- * @throws CarbonUtilException if any problem while reading
+ * @throws IOException if any problem while reading
*/
public static List<DataFileFooter> readCarbonIndexFile(String taskId, String bucketNumber,
List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier absoluteTableIdentifier)
- throws CarbonUtilException {
+ throws IOException {
// need to sort the block info list based for task in ascending order so
// it will be sinkup with block index read from file
Collections.sort(tableBlockInfoList);
@@ -1176,12 +1124,8 @@ public final class CarbonUtil {
.getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
bucketNumber);
DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
- try {
- // read the index info and return
- return fileFooterConverter.getIndexInfo(carbonIndexFilePath, tableBlockInfoList);
- } catch (IOException e) {
- throw new CarbonUtilException("Problem while reading the file metadata", e);
- }
+ // read the index info and return
+ return fileFooterConverter.getIndexInfo(carbonIndexFilePath, tableBlockInfoList);
}
/**
@@ -1208,7 +1152,7 @@ public final class CarbonUtil {
* @param csvFilePath
* @return
*/
- public static String readHeader(String csvFilePath) {
+ public static String readHeader(String csvFilePath) throws IOException {
DataInputStream fileReader = null;
BufferedReader bufferedReader = null;
@@ -1220,11 +1164,6 @@ public final class CarbonUtil {
bufferedReader = new BufferedReader(new InputStreamReader(fileReader,
Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
readLine = bufferedReader.readLine();
-
- } catch (FileNotFoundException e) {
- LOGGER.error(e, "CSV Input File not found " + e.getMessage());
- } catch (IOException e) {
- LOGGER.error(e, "Not able to read CSV input File " + e.getMessage());
} finally {
CarbonUtil.closeStreams(fileReader, bufferedReader);
}
@@ -1292,17 +1231,12 @@ public final class CarbonUtil {
* @param dataChunkBytes datachunk thrift object in bytes
* @return data chunk thrift object
*/
- public static DataChunk2 readDataChunk(byte[] dataChunkBytes) {
- try {
- return (DataChunk2) read(dataChunkBytes, new ThriftReader.TBaseCreator() {
+ public static DataChunk2 readDataChunk(byte[] dataChunkBytes) throws IOException {
+ return (DataChunk2) read(dataChunkBytes, new ThriftReader.TBaseCreator() {
@Override public TBase create() {
return new DataChunk2();
}
});
- } catch (IOException e) {
- LOGGER.error(e);
- }
- return null;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtilException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtilException.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtilException.java
deleted file mode 100644
index bdeed7f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtilException.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.util;
-
-import java.util.Locale;
-
-public class CarbonUtilException extends Exception {
-
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * The Error message.
- */
- private String msg = "";
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public CarbonUtilException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public CarbonUtilException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
index a65dafb..64c0c53 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
@@ -114,7 +114,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
DIFFNONDECIMAL(3);
private int priority;
- private PRIORITY(int priority) {
+ PRIORITY(int priority) {
this.priority = priority;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 775238d..c9ed234 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -152,7 +152,7 @@ public final class DataTypeUtil {
public static byte[] bigDecimalToByte(BigDecimal num) {
BigInteger sig = new BigInteger(num.unscaledValue().toString());
int scale = num.scale();
- byte[] bscale = new byte[] { (byte) (scale) };
+ byte[] bscale = { (byte) (scale) };
byte[] buff = sig.toByteArray();
byte[] completeArr = new byte[buff.length + bscale.length];
System.arraycopy(bscale, 0, completeArr, 0, bscale.length);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index 98094c5..1b82d70 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -186,7 +186,7 @@ public final class ValueCompressionUtil {
getDataType(Math.pow(10, mantissa) * ((double) maxValue - (double) minValue), 0,
dataTypeSelected);
- CompressionFinder[] finders = new CompressionFinder[] {
+ CompressionFinder[] finders = {
new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, adaptiveDataType, adaptiveDataType,
CompressionFinder.PRIORITY.ACTUAL, measureStoreType),
new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, adaptiveDataType, deltaDataType,
@@ -462,7 +462,7 @@ public final class ValueCompressionUtil {
double[] defaultResult = new double[value.length];
for (double a : value) {
- defaultResult[i] = (double) (maxValue - a);
+ defaultResult[i] = maxValue - a;
i++;
}
return defaultResult;
@@ -508,7 +508,7 @@ public final class ValueCompressionUtil {
long[] longResult = new long[value.length];
for (double a : value) {
- longResult[i] = (long) (Math.round(Math.pow(10, mantissa) * a));
+ longResult[i] = Math.round(Math.pow(10, mantissa) * a);
i++;
}
return longResult;
@@ -587,7 +587,7 @@ public final class ValueCompressionUtil {
for (double a : value) {
BigDecimal val = BigDecimal.valueOf(a);
double diff = max.subtract(val).doubleValue();
- longResult[i] = (long) (Math.round(diff * Math.pow(10, mantissa)));
+ longResult[i] = Math.round(diff * Math.pow(10, mantissa));
i++;
}
return longResult;
@@ -899,7 +899,7 @@ public final class ValueCompressionUtil {
DATA_LONG(),
DATA_BIGINT(),
DATA_DOUBLE();
- private DataType() {
+ DataType() {
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/writer/CarbonFooterWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonFooterWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonFooterWriter.java
index 758c2d9..5a8b98d 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonFooterWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonFooterWriter.java
@@ -48,8 +48,6 @@ public class CarbonFooterWriter {
try {
thriftWriter.write(footer);
thriftWriter.writeOffset(currentPosition);
- } catch (Exception e) {
- throw e;
} finally {
thriftWriter.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java
index bf6fc3b..01577c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileWriter.java
@@ -35,7 +35,6 @@ public class CarbonIndexFileWriter {
/**
* It writes thrift object to file
*
- * @param footer
* @throws IOException
*/
public void writeThrift(TBase indexObject) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/writer/HierarchyValueWriterForCSV.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/HierarchyValueWriterForCSV.java b/core/src/main/java/org/apache/carbondata/core/writer/HierarchyValueWriterForCSV.java
index 61a89f9..f112f61 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/HierarchyValueWriterForCSV.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/HierarchyValueWriterForCSV.java
@@ -104,7 +104,7 @@ public class HierarchyValueWriterForCSV {
/**
* @return Returns the byteArrayList.
*/
- public List<ByteArrayHolder> getByteArrayList() throws KettleException {
+ public List<ByteArrayHolder> getByteArrayList() {
return byteArrayholder;
}
@@ -287,7 +287,7 @@ public class HierarchyValueWriterForCSV {
byteArrayholder.clear();
}
- private void closeStreamAndDeleteFile(File f, Closeable... streams) throws KettleException {
+ private void closeStreamAndDeleteFile(File f, Closeable... streams) {
boolean isDeleted = false;
for (Closeable stream : streams) {
if (null != stream) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriter.java
index e9d7b1d..a5e8891 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriter.java
@@ -19,7 +19,6 @@
package org.apache.carbondata.core.writer.sortindex;
import java.io.Closeable;
-import java.io.IOException;
import java.util.List;
/**
@@ -32,17 +31,15 @@ public interface CarbonDictionarySortIndexWriter extends Closeable {
* sortedIndex file in thrif format.
*
* @param sortIndexList list of sortIndex
- * @throws IOException In Case of any I/O errors occurs.
*/
- public void writeSortIndex(List<Integer> sortIndexList) throws IOException;
+ void writeSortIndex(List<Integer> sortIndexList);
/**
* The method is used write the dictionary sortIndexInverted data to columns
* sortedIndex file in thrif format.
*
* @param invertedSortIndexList list of sortIndexInverted
- * @throws IOException In Case of any I/O errors occurs.
*/
- public void writeInvertedSortIndex(List<Integer> invertedSortIndexList) throws IOException;
+ void writeInvertedSortIndex(List<Integer> invertedSortIndexList);
}