You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2018/07/12 06:55:08 UTC
[1/8] carbondata git commit: [CARBONDATA-2720] Remove dead code
Repository: carbondata
Updated Branches:
refs/heads/master d267c40b8 -> f91140367
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
index 768dedb..caf121f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
@@ -42,7 +42,6 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
-import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
@@ -78,10 +77,6 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable());
}
- @Override public DataField[] getOutput() {
- return child.getOutput();
- }
-
@Override public void initialize() throws IOException {
super.initialize();
child.initialize();
@@ -187,7 +182,7 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
if (rowsNotExist) {
rowsNotExist = false;
dataHandler = CarbonFactHandlerFactory
- .createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
+ .createCarbonFactHandler(model);
dataHandler.initialise();
}
processBatch(insideRangeIterator.next(), dataHandler);
@@ -260,10 +255,6 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
rowCounter.getAndAdd(1);
}
- @Override protected CarbonRow processRow(CarbonRow row) {
- return null;
- }
-
@Override public void close() {
if (!closed) {
super.close();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepImpl.java
index 9521db4..c9f5fcc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepImpl.java
@@ -85,10 +85,6 @@ public class InputProcessorStepImpl extends AbstractDataLoadProcessorStep {
return outIterators;
}
- @Override protected CarbonRow processRow(CarbonRow row) {
- return null;
- }
-
@Override public void close() {
if (!closed) {
super.close();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
index 7c4f161..b6858e1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepWithNoConverterImpl.java
@@ -169,10 +169,6 @@ public class InputProcessorStepWithNoConverterImpl extends AbstractDataLoadProce
return iterators;
}
- @Override protected CarbonRow processRow(CarbonRow row) {
- return null;
- }
-
@Override public void close() {
if (!closed) {
super.close();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/steps/JsonInputProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/JsonInputProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/JsonInputProcessorStepImpl.java
index 892be93..13877f9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/JsonInputProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/JsonInputProcessorStepImpl.java
@@ -21,7 +21,6 @@ import java.util.Iterator;
import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
@@ -72,10 +71,6 @@ public class JsonInputProcessorStepImpl extends AbstractDataLoadProcessorStep {
return outIterators;
}
- @Override protected CarbonRow processRow(CarbonRow row) {
- return null;
- }
-
@Override public void close() {
if (!closed) {
super.close();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/steps/SortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/SortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/SortProcessorStepImpl.java
index 856d68c..60ae8d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/SortProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/SortProcessorStepImpl.java
@@ -19,10 +19,8 @@ package org.apache.carbondata.processing.loading.steps;
import java.io.IOException;
import java.util.Iterator;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
-import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
import org.apache.carbondata.processing.loading.sort.Sorter;
@@ -43,11 +41,6 @@ public class SortProcessorStepImpl extends AbstractDataLoadProcessorStep {
}
@Override
- public DataField[] getOutput() {
- return child.getOutput();
- }
-
- @Override
public void initialize() throws IOException {
super.initialize();
child.initialize();
@@ -63,11 +56,6 @@ public class SortProcessorStepImpl extends AbstractDataLoadProcessorStep {
}
@Override
- protected CarbonRow processRow(CarbonRow row) {
- return null;
- }
-
- @Override
public void close() {
if (!closed) {
super.close();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index a4d3d2b..75a231e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -18,7 +18,6 @@ package org.apache.carbondata.processing.merger;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -388,35 +387,4 @@ public class CarbonCompactionUtil {
}
return restructuredBlockExists;
}
-
- /**
- * This method will check for any restructured block in the blocks selected for compaction
- *
- * @param segmentMapping
- * @param tableLastUpdatedTime
- * @return
- */
- public static boolean checkIfAnyRestructuredBlockExists(Map<String, TaskBlockInfo> segmentMapping,
- long tableLastUpdatedTime) {
- boolean restructuredBlockExists = false;
- for (Map.Entry<String, TaskBlockInfo> taskMap : segmentMapping.entrySet()) {
- String segmentId = taskMap.getKey();
- TaskBlockInfo taskBlockInfo = taskMap.getValue();
- Collection<List<TableBlockInfo>> infoList = taskBlockInfo.getAllTableBlockInfoList();
- for (List<TableBlockInfo> listMetadata : infoList) {
- for (TableBlockInfo blockInfo : listMetadata) {
- // if schema modified timestamp is greater than footer stored schema timestamp,
- // it indicates it is a restructured block
- if (tableLastUpdatedTime > blockInfo.getDetailInfo().getSchemaUpdatedTimeStamp()) {
- restructuredBlockExists = true;
- break;
- }
- }
- }
- if (restructuredBlockExists) {
- break;
- }
- }
- return restructuredBlockExists;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 81031de..78af751 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -1185,11 +1185,10 @@ public final class CarbonDataMergerUtil {
* @throws IOException
*/
public static List<CarbonDataMergerUtilResult> compactBlockDeleteDeltaFiles(String seg,
- String blockName, CarbonTable table,
- SegmentUpdateDetails[] segmentUpdateDetails, Long timestamp) throws IOException {
+ String blockName, CarbonTable table, SegmentUpdateDetails[] segmentUpdateDetails,
+ Long timestamp) throws IOException {
- SegmentUpdateStatusManager segmentUpdateStatusManager =
- new SegmentUpdateStatusManager(table);
+ SegmentUpdateStatusManager segmentUpdateStatusManager = new SegmentUpdateStatusManager(table);
List<CarbonDataMergerUtilResult> resultList = new ArrayList<CarbonDataMergerUtilResult>(1);
@@ -1218,11 +1217,8 @@ public final class CarbonDataMergerUtil {
blockDetails.setDeleteDeltaEndTimestamp(timestamp.toString());
try {
- if (startCompactionDeleteDeltaFiles(deleteFilePathList, blockName, fullBlockFilePath)) {
- blockDetails.setCompactionStatus(true);
- } else {
- blockDetails.setCompactionStatus(false);
- }
+ startCompactionDeleteDeltaFiles(deleteFilePathList, blockName, fullBlockFilePath);
+ blockDetails.setCompactionStatus(true);
resultList.add(blockDetails);
} catch (IOException e) {
LOGGER.error("Compaction of Delete Delta Files failed. The complete file path is "
@@ -1240,7 +1236,7 @@ public final class CarbonDataMergerUtil {
* @param fullBlockFilePath
* @return
*/
- public static Boolean startCompactionDeleteDeltaFiles(List<String> deleteDeltaFiles,
+ public static void startCompactionDeleteDeltaFiles(List<String> deleteDeltaFiles,
String blockName, String fullBlockFilePath) throws IOException {
DeleteDeltaBlockDetails deleteDeltaBlockDetails = null;
@@ -1263,7 +1259,6 @@ public final class CarbonDataMergerUtil {
LOGGER.error("Error while writing compacted delete delta file " + fullBlockFilePath);
throw new IOException();
}
- return true;
}
public static Boolean updateStatusFile(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index ff4e47b..0fc229a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -438,8 +438,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
tempStoreLocation, carbonStoreLocation);
carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId());
setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel);
- dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel,
- CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
+ dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel);
try {
dataHandler.initialise();
} catch (CarbonDataWriterException e) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/partition/impl/DefaultLoadBalancer.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/DefaultLoadBalancer.java b/processing/src/main/java/org/apache/carbondata/processing/partition/impl/DefaultLoadBalancer.java
deleted file mode 100644
index 93c5260..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/DefaultLoadBalancer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.partition.impl;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.processing.partition.Partition;
-
-/**
- * A sample load balancer to distribute the partitions to the available nodes in a round robin mode.
- */
-public class DefaultLoadBalancer {
- private Map<String, List<Partition>> nodeToPartitionMap =
- new HashMap<String, List<Partition>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- private Map<Partition, String> partitionToNodeMap =
- new HashMap<Partition, String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- public DefaultLoadBalancer(List<String> nodes, List<Partition> partitions) {
- //Per form a round robin allocation
- int nodeCount = nodes.size();
-
- int partitioner = 0;
- for (Partition partition : partitions) {
- int nodeindex = partitioner % nodeCount;
- String node = nodes.get(nodeindex);
-
- List<Partition> oldList = nodeToPartitionMap.get(node);
- if (oldList == null) {
- oldList = new ArrayList<Partition>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- nodeToPartitionMap.put(node, oldList);
- }
- oldList.add(partition);
-
- partitionToNodeMap.put(partition, node);
-
- partitioner++;
- }
- }
-
- public String getNodeForPartitions(Partition partition) {
- return partitionToNodeMap.get(partition);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/partition/impl/PartitionMultiFileImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/PartitionMultiFileImpl.java b/processing/src/main/java/org/apache/carbondata/processing/partition/impl/PartitionMultiFileImpl.java
deleted file mode 100644
index c303efa..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/PartitionMultiFileImpl.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.partition.impl;
-
-import java.util.List;
-
-import org.apache.carbondata.processing.partition.Partition;
-
-public class PartitionMultiFileImpl implements Partition {
- private static final long serialVersionUID = -4363447826181193976L;
- private String uniqueID;
- private List<String> folderPath;
-
- public PartitionMultiFileImpl(String uniqueID, List<String> folderPath) {
- this.uniqueID = uniqueID;
- this.folderPath = folderPath;
- }
-
- @Override public String getUniqueID() {
- // TODO Auto-generated method stub
- return uniqueID;
- }
-
- @Override public List<String> getFilesPath() {
- // TODO Auto-generated method stub
- return folderPath;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/partition/impl/SampleDataPartitionerImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/SampleDataPartitionerImpl.java b/processing/src/main/java/org/apache/carbondata/processing/partition/impl/SampleDataPartitionerImpl.java
deleted file mode 100644
index 92bd6ff..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/SampleDataPartitionerImpl.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.partition.impl;
-
-import java.util.List;
-
-import org.apache.carbondata.processing.partition.DataPartitioner;
-import org.apache.carbondata.processing.partition.Partition;
-
-/**
- * Sample partition.
- */
-public class SampleDataPartitionerImpl implements DataPartitioner {
-
- @Override
- public List<Partition> getAllPartitions() {
- return null;
- }
-
- @Override
- public List<Partition> getPartitions() {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
index 9b09269..2dc79a3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
@@ -26,7 +26,6 @@ import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.carbondata.processing.partition.spliter.exception.AlterPartitionSliceException;
import org.apache.carbondata.processing.store.CarbonDataFileAttributes;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerColumnar;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
@@ -80,7 +79,7 @@ public class RowResultProcessor {
this.dataHandler.finish();
}
processStatus = true;
- } catch (AlterPartitionSliceException e) {
+ } catch (CarbonDataWriterException e) {
LOGGER.error(e, e.getMessage());
LOGGER.error("Exception in executing RowResultProcessor" + e.getMessage());
processStatus = false;
@@ -97,12 +96,12 @@ public class RowResultProcessor {
return processStatus;
}
- private void addRow(Object[] carbonTuple) throws AlterPartitionSliceException {
+ private void addRow(Object[] carbonTuple) throws CarbonDataWriterException {
CarbonRow row = WriteStepRowUtil.fromMergerRow(carbonTuple, segmentProperties);
try {
this.dataHandler.addDataToStore(row);
} catch (CarbonDataWriterException e) {
- throw new AlterPartitionSliceException("Exception in adding rows in RowResultProcessor", e);
+ throw new CarbonDataWriterException("Exception in adding rows in RowResultProcessor", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/exception/AlterPartitionSliceException.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/exception/AlterPartitionSliceException.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/exception/AlterPartitionSliceException.java
deleted file mode 100644
index 21b53cf..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/exception/AlterPartitionSliceException.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.partition.spliter.exception;
-
-import java.util.Locale;
-
-public class AlterPartitionSliceException extends Exception {
-
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * The Error message.
- */
- private String msg = "";
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public AlterPartitionSliceException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public AlterPartitionSliceException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/splits/TableSplit.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/splits/TableSplit.java b/processing/src/main/java/org/apache/carbondata/processing/splits/TableSplit.java
deleted file mode 100644
index c7d5dd8..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/splits/TableSplit.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.splits;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.processing.partition.Partition;
-
-import org.apache.hadoop.io.Writable;
-
-
-/**
- * It represents one region server as one split.
- */
-public class TableSplit implements Serializable, Writable {
- private static final long serialVersionUID = -8058151330863145575L;
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(TableSplit.class.getName());
- private List<String> locations = new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- private Partition partition;
-
- /**
- * @return the locations
- */
- public List<String> getLocations() {
- return locations;
- }
-
- /**
- * @param locations the locations to set
- */
- public void setLocations(List<String> locations) {
- this.locations = locations;
- }
-
- /**
- * @return Returns the partitions.
- */
- public Partition getPartition() {
- return partition;
- }
-
- /**
- * @param partition The partitions to set.
- */
- public void setPartition(Partition partition) {
- this.partition = partition;
- }
-
- @Override public void readFields(DataInput in) throws IOException {
-
- int sizeLoc = in.readInt();
- for (int i = 0; i < sizeLoc; i++) {
- byte[] b = new byte[in.readInt()];
- in.readFully(b);
- locations.add(new String(b, Charset.defaultCharset()));
- }
-
- byte[] buf = new byte[in.readInt()];
- in.readFully(buf);
- ByteArrayInputStream bis = new ByteArrayInputStream(buf);
- ObjectInputStream ois = new ObjectInputStream(bis);
- try {
- partition = (Partition) ois.readObject();
- } catch (ClassNotFoundException e) {
- LOGGER.error(e, e.getMessage());
- }
- ois.close();
- }
-
- @Override public void write(DataOutput out) throws IOException {
-
- int sizeLoc = locations.size();
- out.writeInt(sizeLoc);
- for (int i = 0; i < sizeLoc; i++) {
- byte[] bytes = locations.get(i).getBytes(Charset.defaultCharset());
- out.writeInt(bytes.length);
- out.write(bytes);
- }
-
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
-
- ObjectOutputStream obs = new ObjectOutputStream(bos);
- obs.writeObject(partition);
- obs.close();
- byte[] byteArray = bos.toByteArray();
- out.writeInt(byteArray.length);
- out.write(byteArray);
- }
-
- public String toString() {
- return partition.getUniqueID() + ' ' + locations;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
index 8bedd80..7861b52 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
@@ -48,10 +48,6 @@ public class CarbonDataFileAttributes {
return taskId;
}
- public void setFactTimeStamp(long factTimeStamp) {
- this.factTimeStamp = factTimeStamp;
- }
-
/**
* @return fact time stamp which is load start time
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index f3cb9c3..97737d0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -34,11 +34,9 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
-import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthEquiSplitGenerator;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
@@ -86,7 +84,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
private ExecutorService consumerExecutorService;
private List<Future<Void>> consumerExecutorServiceTaskList;
private List<CarbonRow> dataRows;
- private ColumnGroupModel colGrpModel;
/**
* semaphore which will used for managing node holder objects
*/
@@ -136,7 +133,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
private void initParameters(CarbonFactDataHandlerModel model) {
- this.colGrpModel = model.getSegmentProperties().getColumnGroupModel();
this.numberOfCores = model.getNumberOfCores();
blockletProcessingCount = new AtomicInteger(0);
producerExecutorService = Executors.newFixedThreadPool(model.getNumberOfCores(),
@@ -358,19 +354,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
int dimSet =
Integer.parseInt(CarbonCommonConstants.DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE);
// if at least one dimension is present then initialize column splitter otherwise null
- int noOfColStore = colGrpModel.getNoOfColumnStore();
- int[] keyBlockSize = new int[noOfColStore + getExpandedComplexColsCount()];
-
- if (model.getDimLens().length > 0) {
- //Using Variable length variable split generator
- //This will help in splitting mdkey to columns. variable split is required because all
- // columns which are part of
- //row store will be in single column store
- //e.g if {0,1,2,3,4,5} is dimension and {0,1,2) is row store dimension
- //than below splitter will return column as {0,1,2}{3}{4}{5}
- ColumnarSplitter columnarSplitter = model.getSegmentProperties().getFixedLengthKeySplitter();
- System.arraycopy(columnarSplitter.getBlockKeySize(), 0, keyBlockSize, 0, noOfColStore);
- }
+ int[] keyBlockSize = new int[getExpandedComplexColsCount()];
// agg type
List<Integer> otherMeasureIndexList =
@@ -398,8 +382,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
int[] blockKeySize = getBlockKeySizeWithComplexTypes(new MultiDimKeyVarLengthEquiSplitGenerator(
CarbonUtil.getIncrementedCardinalityFullyFilled(model.getDimLens().clone()), (byte) dimSet)
.getBlockKeySize());
- System.arraycopy(blockKeySize, noOfColStore, keyBlockSize, noOfColStore,
- blockKeySize.length - noOfColStore);
+ System.arraycopy(blockKeySize, 0, keyBlockSize, 0, blockKeySize.length);
this.dataWriter = getFactDataWriter();
// initialize the channel;
this.dataWriter.initializeWriter();
@@ -413,8 +396,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
*/
private int[] getBlockKeySizeWithComplexTypes(int[] primitiveBlockKeySize) {
int allColsCount = getExpandedComplexColsCount();
- int[] blockKeySizeWithComplexTypes =
- new int[this.colGrpModel.getNoOfColumnStore() + allColsCount];
+ int[] blockKeySizeWithComplexTypes = new int[allColsCount];
List<Integer> blockKeySizeWithComplex =
new ArrayList<Integer>(blockKeySizeWithComplexTypes.length);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 266c75d..63e47f0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -35,7 +35,6 @@ import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonProperties;
@@ -88,10 +87,6 @@ public class CarbonFactDataHandlerModel {
* local store location
*/
private String[] storeLocation;
- /**
- * flag to check whether use inverted index
- */
- private boolean[] isUseInvertedIndex;
/**
* length of each dimension, including dictionary, nodictioncy, complex dimension
@@ -185,8 +180,6 @@ public class CarbonFactDataHandlerModel {
int taskExtension, DataMapWriterListener listener) {
CarbonTableIdentifier identifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
- boolean[] isUseInvertedIndex =
- CarbonDataProcessorUtil.getIsUseInvertedIndex(configuration.getDataFields());
int[] dimLensWithComplex = configuration.getCardinalityFinder().getCardinality();
if (!configuration.isSortTable()) {
@@ -262,7 +255,6 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.setPrimitiveDimLens(simpleDimsLen);
carbonFactDataHandlerModel.setCarbonDataFileAttributes(carbonDataFileAttributes);
carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
- carbonFactDataHandlerModel.setIsUseInvertedIndex(isUseInvertedIndex);
carbonFactDataHandlerModel.setBlockSizeInMB(carbonTable.getBlockSizeInMB());
carbonFactDataHandlerModel.setComplexDimensionKeyGenerator(
configuration.createKeyGeneratorForComplexDimension());
@@ -333,13 +325,6 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes);
CarbonUtil.checkAndCreateFolderWithPermission(carbonDataDirectoryPath);
carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
- List<CarbonDimension> dimensionByTableName = carbonTable.getDimensionByTableName(tableName);
- boolean[] isUseInvertedIndexes = new boolean[dimensionByTableName.size()];
- int index = 0;
- for (CarbonDimension dimension : dimensionByTableName) {
- isUseInvertedIndexes[index++] = dimension.isUseInvertedIndex();
- }
- carbonFactDataHandlerModel.setIsUseInvertedIndex(isUseInvertedIndexes);
carbonFactDataHandlerModel.setPrimitiveDimLens(segmentProperties.getDimColumnsCardinality());
carbonFactDataHandlerModel.setBlockSizeInMB(carbonTable.getBlockSizeInMB());
@@ -528,13 +513,6 @@ public class CarbonFactDataHandlerModel {
isCompactionFlow = compactionFlow;
}
- public boolean[] getIsUseInvertedIndex() {
- return isUseInvertedIndex;
- }
-
- public void setIsUseInvertedIndex(boolean[] isUseInvertedIndex) {
- this.isUseInvertedIndex = isUseInvertedIndex;
- }
/**
*
* @return segmentProperties
@@ -621,10 +599,6 @@ public class CarbonFactDataHandlerModel {
return count;
}
- public boolean isSortColumn(int columnIndex) {
- return columnIndex < segmentProperties.getNumberOfSortColumns();
- }
-
public TableSpec getTableSpec() {
return tableSpec;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactHandlerFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactHandlerFactory.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactHandlerFactory.java
index b4d18f4..79a8a86 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactHandlerFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactHandlerFactory.java
@@ -28,19 +28,8 @@ public final class CarbonFactHandlerFactory {
* @param handlerType
* @return
*/
- public static CarbonFactHandler createCarbonFactHandler(CarbonFactDataHandlerModel model,
- FactHandlerType handlerType) {
- switch (handlerType) {
- case COLUMNAR:
- return new CarbonFactDataHandlerColumnar(model);
- default:
- return new CarbonFactDataHandlerColumnar(model);
- }
+ public static CarbonFactHandler createCarbonFactHandler(CarbonFactDataHandlerModel model) {
+ return new CarbonFactDataHandlerColumnar(model);
}
-
- public enum FactHandlerType {
- COLUMNAR
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 10888f6..dfe0e24 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -33,7 +33,6 @@ import org.apache.carbondata.common.constants.LoggerAction;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.ColumnType;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
@@ -397,7 +396,7 @@ public final class CarbonDataProcessorUtil {
public static boolean isHeaderValid(String tableName, String[] csvHeader,
CarbonDataLoadSchema schema, List<String> ignoreColumns) {
Iterator<String> columnIterator =
- CarbonDataProcessorUtil.getSchemaColumnNames(schema, tableName).iterator();
+ CarbonDataProcessorUtil.getSchemaColumnNames(schema).iterator();
Set<String> csvColumns = new HashSet<String>(csvHeader.length);
Collections.addAll(csvColumns, csvHeader);
@@ -418,28 +417,17 @@ public final class CarbonDataProcessorUtil {
* @param schema
* @param tableName
*/
- public static Set<String> getSchemaColumnNames(CarbonDataLoadSchema schema, String tableName) {
+ public static Set<String> getSchemaColumnNames(CarbonDataLoadSchema schema) {
Set<String> columnNames = new HashSet<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
String factTableName = schema.getCarbonTable().getTableName();
- if (tableName.equals(factTableName)) {
- List<CarbonDimension> dimensions =
- schema.getCarbonTable().getDimensionByTableName(factTableName);
- for (CarbonDimension dimension : dimensions) {
- columnNames.add(dimension.getColName());
- }
- List<CarbonMeasure> measures = schema.getCarbonTable().getMeasureByTableName(factTableName);
- for (CarbonMeasure msr : measures) {
- columnNames.add(msr.getColName());
- }
- } else {
- List<CarbonDimension> dimensions = schema.getCarbonTable().getDimensionByTableName(tableName);
- for (CarbonDimension dimension : dimensions) {
- columnNames.add(dimension.getColName());
- }
- List<CarbonMeasure> measures = schema.getCarbonTable().getMeasureByTableName(tableName);
- for (CarbonMeasure msr : measures) {
- columnNames.add(msr.getColName());
- }
+ List<CarbonDimension> dimensions =
+ schema.getCarbonTable().getDimensionByTableName(factTableName);
+ for (CarbonDimension dimension : dimensions) {
+ columnNames.add(dimension.getColName());
+ }
+ List<CarbonMeasure> measures = schema.getCarbonTable().getMeasureByTableName(factTableName);
+ for (CarbonMeasure msr : measures) {
+ columnNames.add(msr.getColName());
}
return columnNames;
}
@@ -604,19 +592,6 @@ public final class CarbonDataProcessorUtil {
}
/**
- * This method will return a flag based on whether a column is applicable for RLE encoding
- *
- * @param dimensionType
- * @return
- */
- public static boolean isRleApplicableForColumn(ColumnType dimensionType) {
- if (dimensionType == ColumnType.GLOBAL_DICTIONARY) {
- return true;
- }
- return false;
- }
-
- /**
* This method will return an array whose element with be appended with the `append` strings
* @param inputArr inputArr
* @param append strings to append
[5/8] carbondata git commit: [CARBONDATA-2720] Remove dead code
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java
deleted file mode 100644
index 19e91da..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertNull;
-
-public class SegmentTaskIndexStoreTest {
-
- private static short version = 1;
- private static String locations[] = { "/tmp" };
- private static SegmentTaskIndexStore taskIndexStore;
- private static TableBlockInfo tableBlockInfo;
- private static AbsoluteTableIdentifier absoluteTableIdentifier;
-
- @BeforeClass public static void setUp() {
- CacheProvider cacheProvider = CacheProvider.getInstance();
- taskIndexStore = (SegmentTaskIndexStore) cacheProvider.
- <TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper>
- createCache(CacheType.DRIVER_BTREE);
- tableBlockInfo = new TableBlockInfo("file", 0L, "SG100", locations, 10L,
- ColumnarFormatVersion.valueOf(version), null);
- absoluteTableIdentifier = AbsoluteTableIdentifier.from("/tmp",
- new CarbonTableIdentifier("testdatabase", "testtable", "TB100"));
- }
-
- private List<DataFileFooter> getDataFileFooters() {
- SegmentInfo segmentInfo = new SegmentInfo();
- DataFileFooter footer = new DataFileFooter();
- ColumnSchema columnSchema = new ColumnSchema();
- BlockletInfo blockletInfo = new BlockletInfo();
- List<DataFileFooter> footerList = new ArrayList<DataFileFooter>();
- List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
-
- columnSchema.setColumnName("employeeName");
- columnSchemaList.add(new ColumnSchema());
-
- footer.setSegmentInfo(segmentInfo);
- footer.setColumnInTable(columnSchemaList);
- footer.setBlockletList(Arrays.asList(blockletInfo));
- footerList.add(footer);
- return footerList;
- }
-
- @Test public void checkExistenceOfSegmentBTree() {
- TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier =
- new TableSegmentUniqueIdentifier(absoluteTableIdentifier, "SG100");
- SegmentTaskIndexWrapper segmentTaskIndexWrapper =
- taskIndexStore.getIfPresent(tableSegmentUniqueIdentifier);
- Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> result = segmentTaskIndexWrapper != null ?
- segmentTaskIndexWrapper.getTaskIdToTableSegmentMap() :
- null;
- assertNull(result);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
index 20036ec..68bf896 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
@@ -89,13 +89,13 @@ public class SegmentPropertiesTest extends TestCase {
dimensionOrdinalToBlockMapping.put(0, 0);
dimensionOrdinalToBlockMapping.put(1, 1);
dimensionOrdinalToBlockMapping.put(2, 2);
- dimensionOrdinalToBlockMapping.put(3, 2);
- dimensionOrdinalToBlockMapping.put(4, 3);
- dimensionOrdinalToBlockMapping.put(5, 4);
- dimensionOrdinalToBlockMapping.put(6, 4);
- dimensionOrdinalToBlockMapping.put(7, 4);
- dimensionOrdinalToBlockMapping.put(8, 5);
- dimensionOrdinalToBlockMapping.put(9, 6);
+ dimensionOrdinalToBlockMapping.put(3, 3);
+ dimensionOrdinalToBlockMapping.put(4, 4);
+ dimensionOrdinalToBlockMapping.put(5, 5);
+ dimensionOrdinalToBlockMapping.put(6, 6);
+ dimensionOrdinalToBlockMapping.put(7, 7);
+ dimensionOrdinalToBlockMapping.put(8, 8);
+ dimensionOrdinalToBlockMapping.put(9, 9);
Map<Integer, Integer> dimensionOrdinalToBlockMappingActual =
blockMetadataInfos.getDimensionOrdinalToChunkMapping();
assertEquals(dimensionOrdinalToBlockMapping.size(),
@@ -143,7 +143,7 @@ public class SegmentPropertiesTest extends TestCase {
}
@Test public void testEachColumnValueSizeHasProperValue() {
- int[] size = { 1, -1, 2, -1, 3 };
+ int[] size = { 1, -1, 1, 1, -1, 1, 1, 1 };
int[] eachDimColumnValueSize = blockMetadataInfos.getEachDimColumnValueSize();
boolean isEqual = false;
for (int i = 0; i < size.length; i++) {
@@ -170,7 +170,6 @@ public class SegmentPropertiesTest extends TestCase {
private ColumnSchema getDimensionColumn1() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -185,7 +184,6 @@ public class SegmentPropertiesTest extends TestCase {
private ColumnSchema getDimensionColumn2() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI1");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -199,7 +197,6 @@ public class SegmentPropertiesTest extends TestCase {
private ColumnSchema getDimensionColumn3() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI2");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -208,14 +205,12 @@ public class SegmentPropertiesTest extends TestCase {
new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
- dimColumn.setColumnGroup(0);
dimColumn.setNumberOfChild(0);
return dimColumn;
}
private ColumnSchema getDimensionColumn4() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI3");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -225,13 +220,11 @@ public class SegmentPropertiesTest extends TestCase {
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
dimColumn.setNumberOfChild(0);
- dimColumn.setColumnGroup(0);
return dimColumn;
}
private ColumnSchema getDimensionColumn5() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI4");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -245,7 +238,6 @@ public class SegmentPropertiesTest extends TestCase {
private ColumnSchema getDimensionColumn9() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI9");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -254,14 +246,12 @@ public class SegmentPropertiesTest extends TestCase {
new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
- dimColumn.setColumnGroup(1);
dimColumn.setNumberOfChild(0);
return dimColumn;
}
private ColumnSchema getDimensionColumn10() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI10");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -271,13 +261,11 @@ public class SegmentPropertiesTest extends TestCase {
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
dimColumn.setNumberOfChild(0);
- dimColumn.setColumnGroup(1);
return dimColumn;
}
private ColumnSchema getDimensionColumn11() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI11");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -287,13 +275,11 @@ public class SegmentPropertiesTest extends TestCase {
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
dimColumn.setNumberOfChild(0);
- dimColumn.setColumnGroup(1);
return dimColumn;
}
private ColumnSchema getDimensionColumn6() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI5");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.createDefaultArrayType());
@@ -308,7 +294,6 @@ public class SegmentPropertiesTest extends TestCase {
private ColumnSchema getDimensionColumn7() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI6");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
index 9da5a53..2404498 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
@@ -52,7 +52,6 @@ public class SegmentPropertiesTestUtil {
public static ColumnSchema getDimensionColumn1() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -67,7 +66,6 @@ public class SegmentPropertiesTestUtil {
public static ColumnSchema getDimensionColumn2() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI1");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -81,7 +79,6 @@ public class SegmentPropertiesTestUtil {
public static ColumnSchema getDimensionColumn3() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI2");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -90,14 +87,12 @@ public class SegmentPropertiesTestUtil {
new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
- dimColumn.setColumnGroup(0);
dimColumn.setNumberOfChild(0);
return dimColumn;
}
public static ColumnSchema getDimensionColumn4() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI3");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -107,13 +102,11 @@ public class SegmentPropertiesTestUtil {
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
dimColumn.setNumberOfChild(0);
- dimColumn.setColumnGroup(0);
return dimColumn;
}
public static ColumnSchema getDimensionColumn5() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI4");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -127,7 +120,6 @@ public class SegmentPropertiesTestUtil {
public static ColumnSchema getDimensionColumn9() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI9");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -136,14 +128,12 @@ public class SegmentPropertiesTestUtil {
new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
- dimColumn.setColumnGroup(1);
dimColumn.setNumberOfChild(0);
return dimColumn;
}
public static ColumnSchema getDimensionColumn10() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI10");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -153,13 +143,11 @@ public class SegmentPropertiesTestUtil {
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
dimColumn.setNumberOfChild(0);
- dimColumn.setColumnGroup(1);
return dimColumn;
}
public static ColumnSchema getDimensionColumn11() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(false);
dimColumn.setColumnName("IMEI11");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -169,13 +157,11 @@ public class SegmentPropertiesTestUtil {
encodeList.add(Encoding.DICTIONARY);
dimColumn.setEncodingList(encodeList);
dimColumn.setNumberOfChild(0);
- dimColumn.setColumnGroup(1);
return dimColumn;
}
public static ColumnSchema getDimensionColumn6() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI5");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.createDefaultArrayType());
@@ -190,7 +176,6 @@ public class SegmentPropertiesTestUtil {
public static ColumnSchema getDimensionColumn7() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI6");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
deleted file mode 100644
index 4c08c6b..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.block;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
-import org.apache.carbondata.core.datastore.impl.btree.BlockBTreeBuilder;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
-import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertEquals;
-
-public class SegmentTaskIndexTest {
-
- private static SegmentInfo segmentInfo;
- private static DataFileFooter footer;
- private static ColumnSchema columnSchema;
- private static BlockletInfo blockletInfo;
- private static BlockletIndex blockletIndex;
- private static List<DataFileFooter> footerList = new ArrayList<DataFileFooter>();
- private static List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
-
- @BeforeClass public static void setUp() {
- segmentInfo = new SegmentInfo();
- footer = new DataFileFooter();
- columnSchema = new ColumnSchema();
- blockletInfo = new BlockletInfo();
- blockletIndex = new BlockletIndex();
- }
-
- @Test public void testBuild() {
- new MockUp<BlockBTreeBuilder>() {
- @Mock public void build(BTreeBuilderInfo segmentBuilderInfos) {}
- };
- long numberOfRows = 100;
- columnSchema.setColumnName("employeeName");
- columnSchemaList.add(new ColumnSchema());
-
- footer.setSegmentInfo(segmentInfo);
- footer.setColumnInTable(columnSchemaList);
- footer.setBlockletList(Arrays.asList(blockletInfo));
- footer.setNumberOfRows(numberOfRows);
- footerList.add(footer);
-
- SegmentProperties properties = new SegmentProperties(footerList.get(0).getColumnInTable(),
- footerList.get(0).getSegmentInfo().getColumnCardinality());
- SegmentTaskIndex segmentTaskIndex = new SegmentTaskIndex(properties);
- segmentTaskIndex.buildIndex(footerList);
- assertEquals(footerList.get(0).getNumberOfRows(), numberOfRows);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
index f327ef6..8dabe4c 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
@@ -21,8 +21,6 @@ import static org.junit.Assert.assertEquals;
import java.util.Arrays;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -44,11 +42,8 @@ public class FixedLengthDimensionDataChunkTest {
}
@Test public void fillChunkDataTest() {
- KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
int[] maskByteRanges = { 1, 2, 4, 6, 5 };
- keyStructureInfo.setMaskByteRanges(maskByteRanges);
- keyStructureInfo.setMaxKey("1234567".getBytes());
- int res = fixedLengthDimensionDataChunk.fillRawData(0, 0, data, keyStructureInfo);
+ int res = fixedLengthDimensionDataChunk.fillRawData(0, 0, data);
int expectedResult = 4 ;
assertEquals(res, expectedResult);
}
@@ -61,8 +56,7 @@ public class FixedLengthDimensionDataChunkTest {
@Test public void fillConvertedChunkDataTest() {
int[] row = { 1, 2, 4, 6 };
- KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
- int res = fixedLengthDimensionDataChunk.fillSurrogateKey(1, 0, row, keyStructureInfo);
+ int res = fixedLengthDimensionDataChunk.fillSurrogateKey(1, 0, row);
int expectedResult = 1;
assertEquals(res, expectedResult);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java b/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java
deleted file mode 100644
index 8976b2e..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.page.encoding;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.carbondata.core.datastore.ColumnType;
-import org.apache.carbondata.core.datastore.TableSpec;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec;
-import org.apache.carbondata.core.datastore.page.encoding.rle.RLEEncoderMeta;
-import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class RLECodecSuite {
-
- static class TestData {
- private byte[] inputByteData;
- private ColumnPage inputBytePage;
- private byte[] expectedEncodedByteData;
-
- TestData(byte[] inputByteData, byte[] expectedEncodedByteData) throws IOException, MemoryException {
- this.inputByteData = inputByteData;
- inputBytePage = ColumnPage.newPage(
- TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE),
- DataTypes.BYTE, inputByteData.length);
- inputBytePage.setStatsCollector(PrimitivePageStatsCollector.newInstance(DataTypes.BYTE));
- for (int i = 0; i < inputByteData.length; i++) {
- inputBytePage.putData(i, inputByteData[i]);
- }
- this.expectedEncodedByteData = expectedEncodedByteData;
- }
-
- }
-
- private static TestData data1;
- private static TestData data2;
- private static TestData data3;
-
- @BeforeClass public static void setUp() throws IOException, MemoryException {
- setUpData1();
- setUpData2();
- setUpData3();
- }
-
- private static void setUpData1() throws IOException, MemoryException {
- byte[] inputData = new byte[]{10, 10, 3, 4, 5, 6, 7, 7, 7, 7};
- ByteArrayOutputStream bao = new ByteArrayOutputStream();
- DataOutputStream stream = new DataOutputStream(bao);
- stream.writeShort(2);
- stream.writeByte(10);
- stream.writeShort(5|0x8000);
- stream.writeByte(3);
- stream.writeByte(4);
- stream.writeByte(5);
- stream.writeByte(6);
- stream.writeByte(7);
- stream.writeShort(3);
- stream.writeByte(7);
- byte[] expectedEncodedByteData = bao.toByteArray();
- data1 = new TestData(
- inputData,
- expectedEncodedByteData);
- }
-
- private static void setUpData2() throws IOException, MemoryException {
- byte[] inputData = new byte[]{1, 2, 3, 4, 5, 6};
- ByteArrayOutputStream bao = new ByteArrayOutputStream();
- DataOutputStream stream = new DataOutputStream(bao);
- stream.writeShort(6|0x8000);
- stream.writeByte(1);
- stream.writeByte(2);
- stream.writeByte(3);
- stream.writeByte(4);
- stream.writeByte(5);
- stream.writeByte(6);
- byte[] expectedEncodedByteData = bao.toByteArray();
- data2 = new TestData(
- inputData,
- expectedEncodedByteData);
- }
-
- private static void setUpData3() throws IOException, MemoryException {
- byte[] inputData = new byte[]{10, 10, 10, 10, 10, 10};
- ByteArrayOutputStream bao = new ByteArrayOutputStream();
- DataOutputStream stream = new DataOutputStream(bao);
- stream.writeShort(6);
- stream.writeByte(10);
- byte[] expectedEncodedByteData = bao.toByteArray();
- data3 = new TestData(
- inputData,
- expectedEncodedByteData);
- }
-
- private void testBytePageEncode(ColumnPage inputPage, byte[] expectedEncodedBytes)
- throws IOException, MemoryException {
- RLECodec codec = new RLECodec();
- ColumnPageEncoder encoder = codec.createEncoder(null);
- EncodedColumnPage result = encoder.encode(inputPage);
- byte[] encoded = result.getEncodedData().array();
- assertEquals(expectedEncodedBytes.length, encoded.length);
- for (int i = 0; i < encoded.length; i++) {
- assertEquals(expectedEncodedBytes[i], encoded[i]);
- }
- }
-
- private void testBytePageDecode(byte[] inputBytes, byte[] expectedDecodedBytes) throws IOException, MemoryException {
- RLECodec codec = new RLECodec();
- RLEEncoderMeta meta = new RLEEncoderMeta(
- TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE),
- DataTypes.BYTE, expectedDecodedBytes.length, null);
- ColumnPageDecoder decoder = codec.createDecoder(meta);
- ColumnPage page = decoder.decode(inputBytes, 0, inputBytes.length);
- byte[] decoded = page.getBytePage();
- assertEquals(expectedDecodedBytes.length, decoded.length);
- for (int i = 0; i < decoded.length; i++) {
- assertEquals(expectedDecodedBytes[i], decoded[i]);
- }
- }
-
- @Test public void testBytePageEncode() throws MemoryException, IOException {
- testBytePageEncode(data1.inputBytePage, data1.expectedEncodedByteData);
- testBytePageEncode(data2.inputBytePage, data2.expectedEncodedByteData);
- testBytePageEncode(data3.inputBytePage, data3.expectedEncodedByteData);
- }
-
- @Test public void testBytePageDecode() throws IOException, MemoryException {
- testBytePageDecode(data1.expectedEncodedByteData, data1.inputByteData);
- testBytePageDecode(data2.expectedEncodedByteData, data2.inputByteData);
- testBytePageDecode(data3.expectedEncodedByteData, data3.inputByteData);
- }
-
- @AfterClass public static void tearDown() {
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecTest.java
new file mode 100644
index 0000000..8360e02
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.page.encoding;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.ColumnType;
+import org.apache.carbondata.core.datastore.TableSpec;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec;
+import org.apache.carbondata.core.datastore.page.encoding.rle.RLEEncoderMeta;
+import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class RLECodecTest {
+
+ static class TestData {
+ private byte[] inputByteData;
+ private ColumnPage inputBytePage;
+ private byte[] expectedEncodedByteData;
+
+ TestData(byte[] inputByteData, byte[] expectedEncodedByteData) throws IOException, MemoryException {
+ this.inputByteData = inputByteData;
+ inputBytePage = ColumnPage.newPage(
+ TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE),
+ DataTypes.BYTE, inputByteData.length);
+ inputBytePage.setStatsCollector(PrimitivePageStatsCollector.newInstance(DataTypes.BYTE));
+ for (int i = 0; i < inputByteData.length; i++) {
+ inputBytePage.putData(i, inputByteData[i]);
+ }
+ this.expectedEncodedByteData = expectedEncodedByteData;
+ }
+
+ }
+
+ private static TestData data1;
+ private static TestData data2;
+ private static TestData data3;
+
+ @BeforeClass public static void setUp() throws IOException, MemoryException {
+ setUpData1();
+ setUpData2();
+ setUpData3();
+ }
+
+ private static void setUpData1() throws IOException, MemoryException {
+ byte[] inputData = new byte[]{10, 10, 3, 4, 5, 6, 7, 7, 7, 7};
+ ByteArrayOutputStream bao = new ByteArrayOutputStream();
+ DataOutputStream stream = new DataOutputStream(bao);
+ stream.writeShort(2);
+ stream.writeByte(10);
+ stream.writeShort(5|0x8000);
+ stream.writeByte(3);
+ stream.writeByte(4);
+ stream.writeByte(5);
+ stream.writeByte(6);
+ stream.writeByte(7);
+ stream.writeShort(3);
+ stream.writeByte(7);
+ byte[] expectedEncodedByteData = bao.toByteArray();
+ data1 = new TestData(
+ inputData,
+ expectedEncodedByteData);
+ }
+
+ private static void setUpData2() throws IOException, MemoryException {
+ byte[] inputData = new byte[]{1, 2, 3, 4, 5, 6};
+ ByteArrayOutputStream bao = new ByteArrayOutputStream();
+ DataOutputStream stream = new DataOutputStream(bao);
+ stream.writeShort(6|0x8000);
+ stream.writeByte(1);
+ stream.writeByte(2);
+ stream.writeByte(3);
+ stream.writeByte(4);
+ stream.writeByte(5);
+ stream.writeByte(6);
+ byte[] expectedEncodedByteData = bao.toByteArray();
+ data2 = new TestData(
+ inputData,
+ expectedEncodedByteData);
+ }
+
+ private static void setUpData3() throws IOException, MemoryException {
+ byte[] inputData = new byte[]{10, 10, 10, 10, 10, 10};
+ ByteArrayOutputStream bao = new ByteArrayOutputStream();
+ DataOutputStream stream = new DataOutputStream(bao);
+ stream.writeShort(6);
+ stream.writeByte(10);
+ byte[] expectedEncodedByteData = bao.toByteArray();
+ data3 = new TestData(
+ inputData,
+ expectedEncodedByteData);
+ }
+
+ private void testBytePageEncode(ColumnPage inputPage, byte[] expectedEncodedBytes)
+ throws IOException, MemoryException {
+ RLECodec codec = new RLECodec();
+ ColumnPageEncoder encoder = codec.createEncoder(null);
+ EncodedColumnPage result = encoder.encode(inputPage);
+ byte[] encoded = result.getEncodedData().array();
+ assertEquals(expectedEncodedBytes.length, encoded.length);
+ for (int i = 0; i < encoded.length; i++) {
+ assertEquals(expectedEncodedBytes[i], encoded[i]);
+ }
+ }
+
+ private void testBytePageDecode(byte[] inputBytes, byte[] expectedDecodedBytes) throws IOException, MemoryException {
+ RLECodec codec = new RLECodec();
+ RLEEncoderMeta meta = new RLEEncoderMeta(
+ TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE),
+ DataTypes.BYTE, expectedDecodedBytes.length, null);
+ ColumnPageDecoder decoder = codec.createDecoder(meta);
+ ColumnPage page = decoder.decode(inputBytes, 0, inputBytes.length);
+ byte[] decoded = page.getBytePage();
+ assertEquals(expectedDecodedBytes.length, decoded.length);
+ for (int i = 0; i < decoded.length; i++) {
+ assertEquals(expectedDecodedBytes[i], decoded[i]);
+ }
+ }
+
+ @Test public void testBytePageEncode() throws MemoryException, IOException {
+ testBytePageEncode(data1.inputBytePage, data1.expectedEncodedByteData);
+ testBytePageEncode(data2.inputBytePage, data2.expectedEncodedByteData);
+ testBytePageEncode(data3.inputBytePage, data3.expectedEncodedByteData);
+ }
+
+ @Test public void testBytePageDecode() throws IOException, MemoryException {
+ testBytePageDecode(data1.expectedEncodedByteData, data1.inputByteData);
+ testBytePageDecode(data2.expectedEncodedByteData, data2.inputByteData);
+ testBytePageDecode(data3.expectedEncodedByteData, data3.inputByteData);
+ }
+
+ @AfterClass public static void tearDown() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
index 4847177..edefd09 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
@@ -72,7 +72,7 @@ public class DictionaryClientTest {
empColumnSchema.setDimensionColumn(true);
empColumnSchema.setDataType(DataTypes.STRING);
empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
- empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+ empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0);
ageColumnSchema = new ColumnSchema();
ageColumnSchema.setColumnName("ageNameCol");
@@ -80,7 +80,7 @@ public class DictionaryClientTest {
ageColumnSchema.setColumnUniqueId("ageNameCol");
ageColumnSchema.setDimensionColumn(true);
ageColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
- ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0, 0);
+ ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0);
// Create a Table
tableSchema = new TableSchema();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
index 2d31fa5..ef06ac1 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
@@ -62,7 +62,7 @@ public class IncrementalColumnDictionaryGeneratorTest {
String storePath = System.getProperty("java.io.tmpdir") + "/tmp";
tableInfo.setTablePath(storePath + "/test" + "/TestTable");
carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
- carbonDimension = new CarbonDimension(columnSchema,0,0,0,0,0);
+ carbonDimension = new CarbonDimension(columnSchema,0,0,0,0);
}
@Test public void generateKeyOnce() throws Exception {
@@ -135,7 +135,7 @@ public class IncrementalColumnDictionaryGeneratorTest {
columnSchema.setColumnName("empNameCol");
columnSchema.setDataType(DataTypes.STRING);
columnSchema.setColumnUniqueId("empNameCol");
- CarbonDimension carbonDimension = new CarbonDimension(columnSchema, 0, 0, 0, 0, 0);
+ CarbonDimension carbonDimension = new CarbonDimension(columnSchema, 0, 0, 0, 0);
// Create the generator and add the keys to dictionary
IncrementalColumnDictionaryGenerator generator =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
index 2564ead..5ff51cc 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
@@ -65,7 +65,7 @@ public class ServerDictionaryGeneratorTest {
empColumnSchema.setColumnUniqueId("empNameCol");
empColumnSchema.setDimensionColumn(true);
empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
- empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+ empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0,0);
ageColumnSchema = new ColumnSchema();
ageColumnSchema.setColumnName("empNameCol");
@@ -73,7 +73,7 @@ public class ServerDictionaryGeneratorTest {
ageColumnSchema.setDataType(DataTypes.SHORT_INT);
ageColumnSchema.setDimensionColumn(true);
ageColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
- ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0, 0);
+ ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0);
// Create a Table
tableSchema = new TableSchema();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
index e40bdad..96573bc 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
@@ -65,7 +65,7 @@ public class TableDictionaryGeneratorTest {
empColumnSchema.setDimensionColumn(true);
empColumnSchema.setDataType(DataTypes.STRING);
empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
- empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+ empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0);
ageColumnSchema = new ColumnSchema();
ageColumnSchema.setDataType(DataTypes.SHORT_INT);
@@ -73,7 +73,7 @@ public class TableDictionaryGeneratorTest {
ageColumnSchema.setColumnUniqueId("ageNameCol");
ageColumnSchema.setDimensionColumn(true);
ageColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
- ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0, 0);
+ ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0);
// Create a Table
tableSchema = new TableSchema();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
index fcef8f4..0ec19f2 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
@@ -117,7 +117,6 @@ public class CarbonMetadataTest {
private static ColumnSchema getColumnarDimensionColumn() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -167,8 +166,8 @@ public class CarbonMetadataTest {
ColumnSchema colSchema2 = new ColumnSchema();
colSchema1.setColumnUniqueId("1");
colSchema2.setColumnUniqueId("2");
- carbonDimensions.add(new CarbonDimension(colSchema1, 1, 1, 2, 1));
- carbonDimensions.add(new CarbonDimension(colSchema2, 2, 2, 2, 2));
+ carbonDimensions.add(new CarbonDimension(colSchema1, 1, 1, 1));
+ carbonDimensions.add(new CarbonDimension(colSchema2, 2, 2, 2));
new MockUp<CarbonTable>() {
@Mock public String getTableName() {
return "carbonTestTable";
@@ -192,12 +191,12 @@ public class CarbonMetadataTest {
ColumnSchema colSchema1 = new ColumnSchema();
ColumnSchema colSchema2 = new ColumnSchema();
colSchema1.setColumnUniqueId("1");
- carbonDimensions.add(new CarbonDimension(colSchema1, 1, 1, 2, 1));
+ carbonDimensions.add(new CarbonDimension(colSchema1, 1, 1, 1));
final List<CarbonDimension> carbonChildDimensions = new ArrayList();
ColumnSchema colSchema3 = new ColumnSchema();
colSchema3.setColumnUniqueId("9");
colSchema2.setColumnUniqueId("2");
- carbonChildDimensions.add(new CarbonDimension(colSchema3, 1, 1, 2, 1));
+ carbonChildDimensions.add(new CarbonDimension(colSchema3, 1, 1, 1));
new MockUp<CarbonTable>() {
@Mock public String getTableName() {
return "carbonTestTable";
@@ -232,13 +231,13 @@ public class CarbonMetadataTest {
ColumnSchema colSchema1 = new ColumnSchema();
colSchema1.setColumnUniqueId("1");
colSchema1.setNumberOfChild(1);
- CarbonDimension carbonDimension = new CarbonDimension(colSchema1, 1, 1, 2, 1);
+ CarbonDimension carbonDimension = new CarbonDimension(colSchema1, 1, 1, 1);
carbonDimensions.add(carbonDimension);
final List<CarbonDimension> carbonChildDimensions = new ArrayList();
ColumnSchema colSchema2 = new ColumnSchema();
colSchema2.setColumnUniqueId("9");
colSchema2.setNumberOfChild(0);
- carbonChildDimensions.add(new CarbonDimension(colSchema2, 1, 1, 2, 1));
+ carbonChildDimensions.add(new CarbonDimension(colSchema2, 1, 1, 1));
new MockUp<CarbonTable>() {
@Mock public String getTableName() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index 522bf41..897c3cf 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -279,18 +279,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -363,18 +355,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -445,18 +429,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -521,18 +497,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -596,18 +564,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -670,18 +630,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -744,18 +696,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -819,18 +763,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -893,18 +829,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -973,18 +901,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -1047,18 +967,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -1122,18 +1034,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -1203,18 +1107,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -1277,18 +1173,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -1362,18 +1250,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -1480,18 +1360,10 @@ public class ThriftWrapperSchemaConverterImplTest {
return "1";
}
- @Mock public boolean isColumnar() {
- return true;
- }
-
@Mock public boolean isDimensionColumn() {
return true;
}
- @Mock public int getColumnGroupId() {
- return 1;
- }
-
@Mock public int getScale() {
return 1;
}
@@ -1566,12 +1438,10 @@ public class ThriftWrapperSchemaConverterImplTest {
ColumnSchema wrapperColumnSchema = new ColumnSchema();
wrapperColumnSchema.setColumnUniqueId("1");
wrapperColumnSchema.setColumnName("columnName");
- wrapperColumnSchema.setColumnar(true);
wrapperColumnSchema.setDataType(DataTypes.STRING);
wrapperColumnSchema.setDimensionColumn(true);
wrapperColumnSchema.setEncodingList(encodings);
wrapperColumnSchema.setNumberOfChild(1);
- wrapperColumnSchema.setColumnGroup(1);
wrapperColumnSchema.setDefaultValue(new byte[] { 1, 2 });
wrapperColumnSchema.setColumnProperties(columnPropertyMap);
wrapperColumnSchema.setInvisible(true);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
index 0266df4..ec1303f 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
@@ -64,13 +64,12 @@ public class CarbonTableTest extends TestCase {
}
@Test public void testDimensionPresentInTableIsProper() {
- CarbonDimension dimension = new CarbonDimension(getColumnarDimensionColumn(), 0, -1, -1,-1);
+ CarbonDimension dimension = new CarbonDimension(getColumnarDimensionColumn(), 0, -1, -1);
assertTrue(carbonTable.getDimensionByName("carbonTestTable", "IMEI").equals(dimension));
}
static ColumnSchema getColumnarDimensionColumn() {
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
index 84312cd..d3403d5 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
@@ -67,7 +67,6 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
List<ColumnSchema> cols = new ArrayList<ColumnSchema>();
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -80,7 +79,6 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
cols.add(dimColumn);
ColumnSchema structColumn = new ColumnSchema();
- structColumn.setColumnar(true);
structColumn.setColumnName("mobile");
structColumn.setColumnUniqueId(UUID.randomUUID().toString());
structColumn.setDataType(DataTypes.createDefaultStructType());
@@ -90,7 +88,6 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
cols.add(structColumn);
ColumnSchema primitiveColumn = new ColumnSchema();
- primitiveColumn.setColumnar(true);
primitiveColumn.setColumnName("mobile.stdcode");
primitiveColumn.setColumnUniqueId(UUID.randomUUID().toString());
primitiveColumn.setDataType(DataTypes.STRING);
@@ -100,7 +97,6 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
cols.add(primitiveColumn);
ColumnSchema arrayColumn = new ColumnSchema();
- arrayColumn.setColumnar(true);
arrayColumn.setColumnName("mobile.val");
arrayColumn.setColumnUniqueId(UUID.randomUUID().toString());
arrayColumn.setDataType(DataTypes.createDefaultArrayType());
@@ -110,7 +106,6 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
cols.add(arrayColumn);
ColumnSchema primitiveColumn1 = new ColumnSchema();
- primitiveColumn1.setColumnar(true);
primitiveColumn1.setColumnName("mobile.val.phoneno");
primitiveColumn1.setColumnUniqueId(UUID.randomUUID().toString());
primitiveColumn1.setDataType(DataTypes.STRING);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
deleted file mode 100644
index be91410..0000000
--- a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.collector.impl;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-
-public class RawBasedResultCollectorTest {
-//
-// private static RawBasedResultCollector rawBasedResultCollector;
-// private static BlockExecutionInfo blockExecutionInfo;
-// private static KeyGenerator keyGenerator;
-//
-// @BeforeClass public static void setUp() {
-// keyGenerator = new MockUp<KeyGenerator>() {
-// @SuppressWarnings("unused") @Mock long[] getKeyArray(byte[] key, int[] maskedByteRanges) {
-// return new long[] { 1, 2 };
-// }
-//
-// @SuppressWarnings("unused") @Mock byte[] generateKey(long[] keys) throws KeyGenException {
-// return new byte[] { 1, 2 };
-// }
-//
-// }.getMockInstance();
-//
-// blockExecutionInfo = new BlockExecutionInfo();
-// KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
-// keyStructureInfo.setKeyGenerator(keyGenerator);
-// keyStructureInfo.setMaxKey(new byte[] { 1, 2 });
-// keyStructureInfo.setMaskedBytes(new int[] { 1, 2 });
-// keyStructureInfo.setMaskByteRanges(new int[] { 1, 2 });
-// blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
-// MeasureInfo aggregatorInfo = new MeasureInfo();
-// aggregatorInfo.setMeasureOrdinals(new int[] { 10, 20, 30, 40 });
-// aggregatorInfo.setMeasureExists(new boolean[] { true, false, false, false });
-// aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
-// aggregatorInfo.setMeasureDataTypes(
-// new DataType[] { DataTypes.INT, DataTypes.TIMESTAMP, DataTypes.INT, DataTypes.INT });
-// ProjectionMeasure queryMeasure1 = new ProjectionMeasure("QMCol1");
-// queryMeasure1.setQueryOrder(1);
-// ProjectionMeasure queryMeasure2 = new ProjectionMeasure("QMCol2");
-// queryMeasure1.setQueryOrder(2);
-// ProjectionMeasure queryMeasure3 = new ProjectionMeasure("QMCol3");
-// queryMeasure1.setQueryOrder(3);
-// ProjectionMeasure queryMeasure4 = new ProjectionMeasure("QMCol4");
-// queryMeasure1.setQueryOrder(4);
-// ProjectionDimension queryDimension1 = new ProjectionDimension("QDCol1");
-// queryDimension1.setQueryOrder(1);
-// ColumnSchema columnSchema = new ColumnSchema();
-// queryDimension1.setDimension(new CarbonDimension(columnSchema, 0, 0, 0, 0));
-// ProjectionDimension queryDimension2 = new ProjectionDimension("QDCol2");
-// queryDimension2.setQueryOrder(2);
-// queryDimension2.setDimension(new CarbonDimension(columnSchema, 1, 1, 1, 1));
-// ProjectionDimension queryDimension3 = new ProjectionDimension("QDCol3");
-// queryDimension3.setQueryOrder(3);
-// queryDimension3.setDimension(new CarbonDimension(columnSchema, 2, 0, 0, 0));
-// ProjectionDimension queryDimension4 = new ProjectionDimension("QDCol4");
-// queryDimension4.setQueryOrder(4);
-// queryDimension4.setDimension(new CarbonDimension(columnSchema, 3, 0, 0, 0));
-// blockExecutionInfo.setProjectionDimensions(
-// new ProjectionDimension[] { queryDimension1, queryDimension2, queryDimension3,
-// queryDimension4 });
-// blockExecutionInfo.setProjectionMeasures(
-// new ProjectionMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
-// blockExecutionInfo.setFixedKeyUpdateRequired(true);
-// blockExecutionInfo.setMeasureInfo(aggregatorInfo);
-// blockExecutionInfo.setMaskedByteForBlock(new int[] { 1, 2 });
-// blockExecutionInfo.setBlockKeyGenerator(keyGenerator);
-// rawBasedResultCollector = new RawBasedResultCollector(blockExecutionInfo);
-// }
-//
-// @Test public void testToCollectData() {
-//
-// new MockUp<NonFilterQueryScannedResult>() {
-// @SuppressWarnings("unused") @Mock byte[] getDictionaryKeyArray() {
-// this.getMockInstance().incrementCounter();
-// return new byte[] { 1, 2 };
-// }
-//
-// @SuppressWarnings("unused") @Mock byte[][] getNoDictionaryKeyArray() {
-// return new byte[][] { { 1, 2 } };
-// }
-//
-// @SuppressWarnings("unused") @Mock byte[][] getComplexTypeKeyArray() {
-// return new byte[][] { { 1, 2 }, { 1, 2 } };
-// }
-//
-// @SuppressWarnings("unused") @Mock public ColumnPage readMeasureChunk(int ordinal) {
-// ColumnPage ColumnPage = new ColumnPage();
-// PresenceMeta presenceMeta = new PresenceMeta();
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// presenceMeta.setBitSet(bitSet);
-// ColumnPage.setNullValueIndexHolder(presenceMeta);
-// CarbonReadDataHolder carbonReadDataHolder = new CarbonReadDataHolder();
-// carbonReadDataHolder.setReadableLongValues(new long[] { 1 });
-// ColumnPage.setColumnPage(carbonReadDataHolder);
-// return ColumnPage;
-// }
-// };
-//
-// new MockUp<QueryUtil>() {
-// @SuppressWarnings("unused") @Mock byte[] getMaskedKey(byte[] data, byte[] maxKey,
-// int[] maskByteRanges, int byteCount) {
-// return new byte[] { 1, 2 };
-// }
-// };
-//
-// BlockletScannedResult abstractScannedResult =
-// new NonFilterQueryScannedResult(blockExecutionInfo);
-// abstractScannedResult.setPageFilteredRowCount(2);
-// List<Object[]> result = rawBasedResultCollector.collectResultInRow(abstractScannedResult, 2);
-// int expectedResult = 2;
-// assertThat(result.size(), is(equalTo(expectedResult)));
-// }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
index e8b75b7..2c9d132 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
@@ -141,42 +141,6 @@ public class QueryUtilTest extends TestCase {
Assert.assertTrue(result);
}
- @Test public void testGetColumnGroupIdWhenOrdinalValueNotPresentInArrayIndex() {
- int ordinal = 0;
- new MockUp<SegmentProperties>() {
- @Mock public int[][] getColumnGroups() {
- return new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
- }
- };
- int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
- int expectedValue = 4; //expectedValue will always be arrayLength - 1
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetColumnGroupIdWhenOrdinalValuePresentInArrayIndex() {
- int ordinal = 1;
- new MockUp<SegmentProperties>() {
- @Mock public int[][] getColumnGroups() {
- return new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
- }
- };
- int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
- int expectedValue = 0;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetColumnGroupIdWhenColumnGroupsIndexValueLengthLessThanOne() {
- int ordinal = 1;
- new MockUp<SegmentProperties>() {
- @Mock public int[][] getColumnGroups() {
- return new int[][] { { 1 } };
- }
- };
- int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
- int expectedValue = -1;
- assertEquals(expectedValue, actualValue);
- }
-
@Test public void testGetMaskedKey() {
byte[] data = { 1, 2, 3, 4, 5, 5 };
byte[] maxKey = { 15, 20, 25, 30, 35, 35 };
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
index 2768e93..7332614 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
@@ -69,19 +69,19 @@ public class RestructureUtilTest {
columnSchema5.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema5.setEncodingList(encodingList);
- CarbonDimension tableBlockDimension1 = new CarbonDimension(columnSchema1, 1, 1, 1, 1);
- CarbonDimension tableBlockDimension2 = new CarbonDimension(columnSchema2, 5, 5, 5, 5);
+ CarbonDimension tableBlockDimension1 = new CarbonDimension(columnSchema1, 1, 1, 1);
+ CarbonDimension tableBlockDimension2 = new CarbonDimension(columnSchema2, 5, 5, 5);
List<CarbonDimension> tableBlockDimensions =
Arrays.asList(tableBlockDimension1, tableBlockDimension2);
- CarbonDimension tableComplexDimension1 = new CarbonDimension(columnSchema3, 4, 4, 4, 4);
- CarbonDimension tableComplexDimension2 = new CarbonDimension(columnSchema4, 2, 2, 2, 2);
+ CarbonDimension tableComplexDimension1 = new CarbonDimension(columnSchema3, 4, 4, 4);
+ CarbonDimension tableComplexDimension2 = new CarbonDimension(columnSchema4, 2, 2, 2);
List<CarbonDimension> tableComplexDimensions =
Arrays.asList(tableComplexDimension1, tableComplexDimension2);
ProjectionDimension queryDimension1 = new ProjectionDimension(tableBlockDimension1);
ProjectionDimension queryDimension2 = new ProjectionDimension(tableComplexDimension2);
- ProjectionDimension queryDimension3 = new ProjectionDimension(new CarbonDimension(columnSchema5, 3, 3, 3, 3));
+ ProjectionDimension queryDimension3 = new ProjectionDimension(new CarbonDimension(columnSchema5, 3, 3, 3));
ProjectionMeasure queryMeasure1 = new ProjectionMeasure(new CarbonMeasure(columnSchema3, 2));
ProjectionMeasure queryMeasure2 = new ProjectionMeasure(new CarbonMeasure(columnSchema4, 4));
List<ProjectionMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/ColumnExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/ColumnExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/ColumnExpressionTest.java
index f2147b2..fea8db0 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/ColumnExpressionTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/ColumnExpressionTest.java
@@ -48,6 +48,7 @@ public class ColumnExpressionTest {
};
ExpressionResult expectedValue = new ExpressionResult(DataTypes.INT, 1);
assertEquals(expectedValue, columnExpression.evaluate(rowImpl));
+ assertEquals(1, rowImpl.size());
}
@Test public void testEvaluateForNullValue() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
index f349fc9..7b20f8a 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
@@ -42,6 +42,8 @@ public class EqualToExpressionUnitTest {
static EqualToExpression equalToExpression;
+ static int i = 0;
+
@Test public void testForEqualToExpressionWithGetString() throws Exception {
ColumnExpression right = new ColumnExpression("name", DataTypes.STRING);
right.setColIndex(0);
@@ -236,6 +238,29 @@ public class EqualToExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForEqualToExpressionWithLeftAndRightDifferentDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataTypes.INT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataTypes.STRING);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row1 = { "String1" };
+ Integer[] row = { 14 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 14;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
@Test public void testEvaluateForEqualToExpressionWithIsNullReturnFalse()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
@@ -309,4 +334,30 @@ public class EqualToExpressionUnitTest {
ExpressionResult result = equalToExpression.evaluate(value);
assertTrue(result.getBoolean());
}
+
+ @Test public void testEvaluateForEqualToExpressionWithDecimalDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ BigDecimal[] row = new BigDecimal[] { new BigDecimal(12345.0) };
+ BigDecimal[] row1 = new BigDecimal[] { new BigDecimal(12346.0) };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public BigDecimal getDecimal() {
+ if (i == 0) {
+ i++;
+ return new BigDecimal(12346.0);
+ } else {
+ return new BigDecimal(12345.0);
+ }
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
index 95e5935..20e077a 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
@@ -88,6 +88,35 @@ public class GreaterThanEqualToExpressionUnitTest {
}
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithShortDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("id", DataTypes.SHORT);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Short[] row1 = { 16 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public Short getShort() {
+ if (isFirst) {
+ isFirst = false;
+ return 15;
+ }
+ return 16;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
@Test public void testEvaluateForGreaterThanEqualToExpressionWithIntDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("right_number", DataTypes.INT);
@@ -111,6 +140,34 @@ public class GreaterThanEqualToExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithIntDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_number", DataTypes.INT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_number", DataTypes.INT);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Integer[] row = { 140 };
+ Integer[] row1 = { 145 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public Integer getInt() {
+ if (isFirst) {
+ isFirst = false;
+ return 140;
+ }
+ return 145;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testEvaluateForGreaterThanEqualToExpressionWithDoubleDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DOUBLE);
@@ -134,6 +191,34 @@ public class GreaterThanEqualToExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithDoubleDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DOUBLE);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_contact", DataTypes.DOUBLE);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Double[] row = { 44D };
+ Double[] row1 = { 45D };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public Double getDouble() {
+ if (isFirst) {
+ isFirst = false;
+ return 44D;
+ }
+ return 45D;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testEvaluateForGreaterThanEqualToExpressionWithLongDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.LONG);
@@ -157,6 +242,34 @@ public class GreaterThanEqualToExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithLongDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.LONG);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataTypes.LONG);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 1234567654321L };
+ Long[] row1 = { 1234567654321L };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public Long getLong() {
+ if (isFirst) {
+ isFirst = false;
+ return 1234567654322L;
+ }
+ return 1234567654321L;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
@Test public void testEvaluateForGreaterThanEqualToExpressionWithTimestampDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
try {
@@ -187,6 +300,41 @@ public class GreaterThanEqualToExpressionUnitTest {
}
}
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithTimestampDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("timestamp", DataTypes.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("timestamp", DataTypes.TIMESTAMP);
+ right.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+ Timestamp[] row1 = { new Timestamp(time) };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public Long getTime() {
+ if (isFirst) {
+ isFirst = false;
+ return 1190505600L;
+ }
+ return 18465213000000L;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
@Test public void testForGreaterThanEqualToExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
@@ -252,6 +400,40 @@ public class GreaterThanEqualToExpressionUnitTest {
}
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithIsNullReturnTrue1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("id", DataTypes.SHORT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
+ right.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public boolean isNull() {
+ if (isFirst) {
+ isFirst = false;
+ return false;
+ }
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
@Test public void testEvaluateForGreaterThanEqualToExpressionWithLeftAndRightDifferentDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("name", DataTypes.STRING);
@@ -275,6 +457,29 @@ public class GreaterThanEqualToExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithLeftAndRightDifferentDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataTypes.INT);
+ left.setColIndex(1);
+ ColumnExpression right = new ColumnExpression("number", DataTypes.STRING);
+ right.setColIndex(0);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row1 = { "String1" };
+ Integer[] row = { 14 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 14;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
@Test public void testForGreaterThanEqualToExpressionWithGetString() throws Exception {
ColumnExpression left = new ColumnExpression("left_name", DataTypes.STRING);
left.setColIndex(0);
[6/8] carbondata git commit: [CARBONDATA-2720] Remove dead code
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 2e87051..2e98f68 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -20,14 +20,11 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
-import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
@@ -37,7 +34,6 @@ import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.keygenerator.KeyGenException;
@@ -55,12 +51,10 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.complextypes.ArrayQueryType;
import org.apache.carbondata.core.scan.complextypes.PrimitiveQueryType;
import org.apache.carbondata.core.scan.complextypes.StructQueryType;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.model.ProjectionDimension;
import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -212,7 +206,7 @@ public class QueryUtil {
*/
public static int[] getDimensionChunkIndexes(List<ProjectionDimension> queryDimensions,
Map<Integer, Integer> dimensionOrdinalToChunkMapping,
- List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions,
+ Set<CarbonDimension> filterDimensions,
Set<Integer> allProjectionListDimensionIndexes) {
// using set as in row group columns will point to same block
Set<Integer> dimensionChunkIndex = new HashSet<Integer>();
@@ -238,13 +232,6 @@ public class QueryUtil {
}
}
}
- for (int i = 0; i < customAggregationDimension.size(); i++) {
- chunkIndex =
- dimensionOrdinalToChunkMapping.get(customAggregationDimension.get(i).getOrdinal());
- // not adding the children dimension as dimension aggregation
- // is not push down in case of complex dimension
- dimensionChunkIndex.add(chunkIndex);
- }
int[] dimensionIndex = ArrayUtils
.toPrimitive(dimensionChunkIndex.toArray(new Integer[dimensionChunkIndex.size()]));
Arrays.sort(dimensionIndex);
@@ -470,58 +457,6 @@ public class QueryUtil {
}
/**
- * Below method will be used to get the mapping of block index and its
- * restructuring info
- *
- * @param queryDimensions query dimension from query model
- * @param segmentProperties segment properties
- * @return map of block index to its restructuring info
- * @throws KeyGenException if problem while key generation
- */
- public static Map<Integer, KeyStructureInfo> getColumnGroupKeyStructureInfo(
- List<ProjectionDimension> queryDimensions, SegmentProperties segmentProperties)
- throws KeyGenException {
- Map<Integer, KeyStructureInfo> rowGroupToItsRSInfo = new HashMap<Integer, KeyStructureInfo>();
- // get column group id and its ordinal mapping of column group
- Map<Integer, List<Integer>> columnGroupAndItsOrdinalMappingForQuery =
- getColumnGroupAndItsOrdinalMapping(queryDimensions);
- Map<Integer, KeyGenerator> columnGroupAndItsKeygenartor =
- segmentProperties.getColumnGroupAndItsKeygenartor();
-
- Iterator<Entry<Integer, List<Integer>>> iterator =
- columnGroupAndItsOrdinalMappingForQuery.entrySet().iterator();
- KeyStructureInfo restructureInfos = null;
- while (iterator.hasNext()) {
- Entry<Integer, List<Integer>> next = iterator.next();
- KeyGenerator keyGenerator = columnGroupAndItsKeygenartor.get(next.getKey());
- restructureInfos = new KeyStructureInfo();
- // sort the ordinal
- List<Integer> ordinal = next.getValue();
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
- //Un sorted
- List<Integer> mdKeyOrdinalForQuery = new ArrayList<Integer>();
- for (Integer ord : ordinal) {
- mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
- mdKeyOrdinalForQuery.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
- }
- Collections.sort(mdKeyOrdinal);
- // get the masked byte range for column group
- int[] maskByteRanges = getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- // max key for column group
- byte[] maxKey = getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils
- .toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()])));
- rowGroupToItsRSInfo
- .put(segmentProperties.getDimensionOrdinalToChunkMapping().get(ordinal.get(0)),
- restructureInfos);
- }
- return rowGroupToItsRSInfo;
- }
-
- /**
* return true if given key is found in array
*
* @param data
@@ -538,69 +473,6 @@ public class QueryUtil {
}
/**
- * Below method will be used to create a mapping of column group columns
- * this mapping will have column group id to all the dimension ordinal
- * present in the column group This mapping will be used during query
- * execution, to create a mask key for the column group dimension which will
- * be used in aggregation and filter query as column group dimension will be
- * stored in bit level
- */
- private static Map<Integer, List<Integer>> getColumnGroupAndItsOrdinalMapping(
- List<ProjectionDimension> origDimensions) {
-
- List<ProjectionDimension> dimensions = new ArrayList<>(origDimensions.size());
- dimensions.addAll(origDimensions);
- /*
- * sort based on column group id
- */
- Collections.sort(dimensions, new Comparator<ProjectionDimension>() {
-
- @Override public int compare(ProjectionDimension o1, ProjectionDimension o2) {
- return Integer
- .compare(o1.getDimension().columnGroupId(), o2.getDimension().columnGroupId());
- }
- });
- // list of row groups this will store all the row group column
- Map<Integer, List<Integer>> columnGroupAndItsOrdinalsMapping =
- new HashMap<Integer, List<Integer>>();
- // to store a column group
- List<Integer> currentColumnGroup = null;
- // current index
- int index = 0;
- // previous column group to check all the column of row id has bee
- // selected
- int prvColumnGroupId = -1;
- while (index < dimensions.size()) {
- // if dimension group id is not zero and it is same as the previous
- // column group id
- // then we need to add ordinal of that column as it belongs to same
- // column group
- if (dimensions.get(index).getDimension().hasEncoding(Encoding.IMPLICIT)) {
- index++;
- continue;
- } else if (!dimensions.get(index).getDimension().isColumnar()
- && dimensions.get(index).getDimension().columnGroupId() == prvColumnGroupId
- && null != currentColumnGroup) {
- currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal());
- }
-
- // if dimension is not a columnar then it is column group column
- else if (!dimensions.get(index).getDimension().isColumnar()) {
- currentColumnGroup = new ArrayList<Integer>();
- columnGroupAndItsOrdinalsMapping
- .put(dimensions.get(index).getDimension().columnGroupId(), currentColumnGroup);
- currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal());
- }
- // update the row id every time,this is required to group the
- // columns
- // of the same row group
- prvColumnGroupId = dimensions.get(index).getDimension().columnGroupId();
- index++;
- }
- return columnGroupAndItsOrdinalsMapping;
- }
-
- /**
* Below method will be used to get masked byte
*
* @param data actual data
@@ -655,52 +527,6 @@ public class QueryUtil {
}
/**
- * Below method will be used to get the key structure for the column group
- *
- * @param segmentProperties segment properties
- * @param dimColumnEvaluatorInfo dimension evaluator info
- * @return key structure info for column group dimension
- * @throws KeyGenException
- */
- public static KeyStructureInfo getKeyStructureInfo(SegmentProperties segmentProperties,
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) throws KeyGenException {
- int colGrpId = getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
- KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
-
- mdKeyOrdinal.add(segmentProperties
- .getColumnGroupMdKeyOrdinal(colGrpId, dimColumnEvaluatorInfo.getColumnIndex()));
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-
- /**
- * Below method will be used to get the column group id based on the ordinal
- *
- * @param segmentProperties segment properties
- * @param ordinal ordinal to be searched
- * @return column group id
- */
- public static int getColumnGroupId(SegmentProperties segmentProperties, int ordinal) {
- int[][] columnGroups = segmentProperties.getColumnGroups();
- int colGrpId = -1;
- for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
- colGrpId++;
- if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
- }
- }
- }
- return colGrpId;
- }
-
- /**
* Below method will be used to get the map of for complex dimension and its type
* which will be used to during query execution to
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java
index 91f2e4e..29b0f13 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.core.scan.expression.exception;
-import java.util.Locale;
/**
* FilterIllegalMemberException class representing exception which can cause while evaluating
@@ -49,40 +48,12 @@ public class FilterIllegalMemberException extends Exception {
/**
* Constructor
*
- * @param msg The error message for this exception.
- */
- public FilterIllegalMemberException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
*/
public FilterIllegalMemberException(Throwable t) {
super(t);
}
/**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
* getMessage
*/
public String getMessage() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java
index a85c52e..969c4fd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.core.scan.expression.exception;
-import java.util.Locale;
-
public class FilterUnsupportedException extends Exception {
/**
@@ -60,24 +58,6 @@ public class FilterUnsupportedException extends Exception {
}
/**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
* getMessage
*/
public String getMessage() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java
index 5e8010c..b01dc08 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java
@@ -64,9 +64,8 @@ public class FalseExpression extends BinaryConditionalExpression {
public ExpressionType getFilterExpressionType() {
return ExpressionType.FALSE;
}
- @Override
- public String getString() {
- return null;
+ @Override public String getString() {
+ return "False(" + (null == left ? null : left.getString());
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java
index 5d09fb1..b49b6f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java
@@ -59,9 +59,8 @@ public class RangeExpression extends BinaryConditionalExpression {
return ExpressionType.RANGE;
}
- @Override
- public String getString() {
- return null;
+ @Override public String getString() {
+ return "Range(" + left.getString() + ',' + right.getString() + ')';
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java
index 3948589..43a05f4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java
@@ -62,9 +62,8 @@ public class TrueExpression extends BinaryConditionalExpression {
public ExpressionType getFilterExpressionType() {
return ExpressionType.TRUE;
}
- @Override
- public String getString() {
- return null;
+ @Override public String getString() {
+ return "True(" + (null == left ? null : left.getString());
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index a99bdee..0587b33 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -239,7 +239,7 @@ public final class FilterUtil {
DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo,
SegmentProperties segmentProperties) {
- if (null != msrColResolvedFilterInfo && msrColResolvedFilterInfo.getMeasure().isColumnar()) {
+ if (null != msrColResolvedFilterInfo && msrColResolvedFilterInfo.getMeasure().isMeasure()) {
CarbonMeasure measuresFromCurrentBlock = segmentProperties
.getMeasureFromCurrentBlock(msrColResolvedFilterInfo.getMeasure().getColumnId());
if (null != measuresFromCurrentBlock) {
@@ -420,7 +420,7 @@ public final class FilterUtil {
MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo,
SegmentProperties segmentProperties) {
- if (null != msrColResolvedFilterInfo && msrColResolvedFilterInfo.getMeasure().isColumnar()) {
+ if (null != msrColResolvedFilterInfo && msrColResolvedFilterInfo.getMeasure().isMeasure()) {
CarbonMeasure measuresFromCurrentBlock = segmentProperties
.getMeasureFromCurrentBlock(msrColResolvedFilterInfo.getMeasure().getColumnId());
if (null != measuresFromCurrentBlock) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 24461a5..78055fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -48,7 +48,7 @@ import org.apache.carbondata.core.util.CarbonUtil;
* for this Range. Also search the data block and set the required bitsets which falls within
* the Range of the RANGE Expression.
*/
-public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
+public class RangeValueFilterExecuterImpl implements FilterExecuter {
private DimColumnResolvedFilterInfo dimColEvaluatorInfo;
private Expression exp;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 1fce7a3..7e5826c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -37,7 +37,6 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -46,8 +45,6 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.MatchExpression;
@@ -427,7 +424,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
} else {
int dictionaryValue = readSurrogatesFromColumnChunk(blockChunkHolder, index, pageIndex,
- dimColumnEvaluatorInfo, dimensionChunkIndex[i]);
+ dimensionChunkIndex[i]);
if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
&& !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
memberString =
@@ -588,46 +585,17 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
*
* @param blockChunkHolder
* @param index
- * @param dimColumnEvaluatorInfo
* @return
*/
private int readSurrogatesFromColumnChunk(RawBlockletColumnChunks blockChunkHolder, int index,
- int page, DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int chunkIndex) {
+ int page, int chunkIndex) {
DimensionColumnPage dataChunk =
blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex].decodeColumnPage(page);
- if (dimColumnEvaluatorInfo.getDimension().isColumnar()) {
- byte[] rawData = dataChunk.getChunkData(index);
- ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
- return CarbonUtil.getSurrogateKey(rawData, byteBuffer);
- } else {
- return readSurrogatesFromColumnGroupBlock(dataChunk, index, dimColumnEvaluatorInfo);
- }
-
+ byte[] rawData = dataChunk.getChunkData(index);
+ ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ return CarbonUtil.getSurrogateKey(rawData, byteBuffer);
}
- /**
- * @param index
- * @param dimColumnEvaluatorInfo
- * @return read surrogate of given row of given column group dimension
- */
- private int readSurrogatesFromColumnGroupBlock(DimensionColumnPage chunk, int index,
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
- try {
- KeyStructureInfo keyStructureInfo =
- QueryUtil.getKeyStructureInfo(segmentProperties, dimColumnEvaluatorInfo);
- byte[] colData = chunk.getChunkData(index);
- long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
- int colGroupId =
- QueryUtil.getColumnGroupId(segmentProperties, dimensionChunkIndex[0]);
- return (int) result[segmentProperties
- .getColumnGroupMdKeyOrdinal(colGroupId, dimensionChunkIndex[0])];
- } catch (KeyGenException e) {
- LOGGER.error(e);
- }
- return 0;
- }
-
-
@Override
public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
deleted file mode 100644
index bc3b39c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
-import org.apache.carbondata.core.util.BitSetGroup;
-
-public class ValueBasedFilterExecuterImpl implements FilterExecuter {
-
- @Override
- public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
- boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
- return new BitSetGroup(0);
- }
-
- @Override
- public boolean applyFilter(RowIntf value, int dimOrdinalMax)
- throws FilterUnsupportedException, IOException {
- throw new FilterUnsupportedException("Unsupported ValueBasedFilterExecuterImpl on row");
- }
-
- @Override
- public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- return new BitSet(1);
- }
-
- @Override
- public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
deleted file mode 100644
index 32b1ae1..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.filter.intf;
-
-import org.apache.carbondata.core.scan.expression.Expression;
-
-public class FilterOptimizerImpl implements FilterOptimizer {
-
- protected Expression filterExpression;
-
- public FilterOptimizerImpl(Expression filterExpression) {
- this.filterExpression = filterExpression;
- }
-
- @Override public Expression optimizeFilter() {
- return filterExpression;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
index bbd33ec..3032bb5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
@@ -19,14 +19,13 @@ package org.apache.carbondata.core.scan.filter.optimizer;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.RangeExpressionEvaluator;
-import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerImpl;
+import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
-public class RangeFilterOptmizer extends FilterOptimizerImpl {
+public class RangeFilterOptmizer implements FilterOptimizer {
RangeExpressionEvaluator rangeExpEvaluator;
public RangeFilterOptmizer(Expression filterExpression) {
- super(filterExpression);
this.rangeExpEvaluator = new RangeExpressionEvaluator(filterExpression);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
index c316b5c..3e58538 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
@@ -44,5 +44,4 @@ public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl {
@Override public FilterExecuterType getFilterExecuterType() {
return FilterExecuterType.TRUE;
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
index eadd502..aac76e8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -36,7 +36,6 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
import org.apache.carbondata.core.mutate.TupleIdEnum;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
@@ -105,16 +104,6 @@ public abstract class BlockletScannedResult {
protected int[] noDictionaryColumnChunkIndexes;
/**
- * column group to is key structure info
- * which will be used to get the key from the complete
- * column group key
- * For example if only one dimension of the column group is selected
- * then from complete column group key it will be used to mask the key and
- * get the particular column key
- */
- protected Map<Integer, KeyStructureInfo> columnGroupKeyStructureInfo;
-
- /**
*
*/
private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
@@ -155,7 +144,6 @@ public abstract class BlockletScannedResult {
this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
this.noDictionaryColumnChunkIndexes = blockExecutionInfo.getNoDictionaryColumnChunkIndexes();
this.dictionaryColumnChunkIndexes = blockExecutionInfo.getDictionaryColumnChunkIndex();
- this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
this.totalDimensionsSize = blockExecutionInfo.getProjectionDimensions().length;
@@ -213,8 +201,7 @@ public abstract class BlockletScannedResult {
int offset = 0;
for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
offset += dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter].fillRawData(
- rowId, offset, completeKey,
- columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ rowId, offset, completeKey);
}
rowCounter++;
return completeKey;
@@ -232,8 +219,7 @@ public abstract class BlockletScannedResult {
int column = 0;
for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
column = dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter]
- .fillSurrogateKey(rowId, column, completeKey,
- columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ .fillSurrogateKey(rowId, column, completeKey);
}
rowCounter++;
return completeKey;
@@ -246,8 +232,7 @@ public abstract class BlockletScannedResult {
int column = 0;
for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
column = dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter]
- .fillVector(vectorInfo, column,
- columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ .fillVector(vectorInfo, column);
}
}
@@ -258,8 +243,7 @@ public abstract class BlockletScannedResult {
int column = 0;
for (int i = 0; i < this.noDictionaryColumnChunkIndexes.length; i++) {
column = dimensionColumnPages[noDictionaryColumnChunkIndexes[i]][pageCounter]
- .fillVector(vectorInfo, column,
- columnGroupKeyStructureInfo.get(noDictionaryColumnChunkIndexes[i]));
+ .fillVector(vectorInfo, column);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
index 7de3e71..26b1135 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
@@ -102,8 +102,7 @@ public class FilterQueryScannedResult extends BlockletScannedResult {
column = dimensionColumnPages[chunkIndex][pageCounter].fillVector(
pageFilteredRowId[pageCounter],
vectorInfo,
- column,
- columnGroupKeyStructureInfo.get(chunkIndex));
+ column);
}
}
@@ -116,8 +115,7 @@ public class FilterQueryScannedResult extends BlockletScannedResult {
column = dimensionColumnPages[chunkIndex][pageCounter].fillVector(
pageFilteredRowId[pageCounter],
vectorInfo,
- column,
- columnGroupKeyStructureInfo.get(chunkIndex));
+ column);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
index 5956595..b5f9d66 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
@@ -72,8 +72,7 @@ public class NonFilterQueryScannedResult extends BlockletScannedResult {
for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
for (int j = 0; j < validRowIds.size(); j++) {
columnDataOffsets[j] += dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter]
- .fillRawData(validRowIds.get(j), columnDataOffsets[j], dictionaryKeyArrayList.get(j),
- columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ .fillRawData(validRowIds.get(j), columnDataOffsets[j], dictionaryKeyArrayList.get(j));
}
}
return dictionaryKeyArrayList;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
index 1665ce6..c3d4df8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
@@ -128,7 +128,6 @@ public class BlockletFullScanner implements BlockletScanner {
}
int lastPageSize = rawBlockletColumnChunks.getDataBlock().numRows()
% CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- ;
if (lastPageSize > 0) {
numberOfRows[numberOfRows.length - 1] = lastPageSize;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
index 3d62d9e..4cce8af 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
@@ -198,13 +198,6 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper>, Serializa
}
/**
- * @return
- */
- public byte[] getImplicitColumnByteArray() {
- return implicitColumnByteArray;
- }
-
- /**
* @param implicitColumnByteArray
*/
public void setImplicitColumnByteArray(byte[] implicitColumnByteArray) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
index 20536fb..1e30fe6 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
@@ -81,14 +81,16 @@ public class DriverQueryStatisticsRecorderImpl implements QueryStatisticsRecorde
* @param statistic
*/
public void recordStatisticsForDriver(QueryStatistic statistic, String queryId) {
- synchronized (lock) {
- // refresh query Statistics Map
- if (queryStatisticsMap.get(queryId) != null) {
- queryStatisticsMap.get(queryId).add(statistic);
- } else {
- List<QueryStatistic> newQueryStatistics = new ArrayList<QueryStatistic>();
- newQueryStatistics.add(statistic);
- queryStatisticsMap.put(queryId, newQueryStatistics);
+ if (null != queryId) {
+ synchronized (lock) {
+ // refresh query Statistics Map
+ if (queryStatisticsMap.get(queryId) != null) {
+ queryStatisticsMap.get(queryId).add(statistic);
+ } else {
+ List<QueryStatistic> newQueryStatistics = new ArrayList<QueryStatistic>();
+ newQueryStatistics.add(statistic);
+ queryStatisticsMap.put(queryId, newQueryStatistics);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
index ed60d37..9635896 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
@@ -36,5 +36,4 @@ public class QueryStatisticsModel {
public Map<String, QueryStatistic> getStatisticsTypeAndObjMap() {
return statisticsTypeAndObjMap;
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 1b43b65..55381fb 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -65,7 +65,7 @@ public class SegmentUpdateStatusManager {
private static final LogService LOG =
LogServiceFactory.getLogService(SegmentUpdateStatusManager.class.getName());
- private AbsoluteTableIdentifier identifier;
+ private final AbsoluteTableIdentifier identifier;
private LoadMetadataDetails[] segmentDetails;
private SegmentUpdateDetails[] updateDetails;
private Map<String, SegmentUpdateDetails> blockAndDetailsMap;
@@ -124,7 +124,7 @@ public class SegmentUpdateStatusManager {
* @param actualBlockName
* @return null if block is not present in segment update status.
*/
- public SegmentUpdateDetails getDetailsForABlock(String segID, String actualBlockName) {
+ private SegmentUpdateDetails getDetailsForABlock(String segID, String actualBlockName) {
String blockIdentifier = CarbonUpdateUtil
.getSegmentBlockNameKey(segID, actualBlockName);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 7cd0c18..f69c1ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -289,7 +289,6 @@ public abstract class AbstractDataFileFooterConverter {
ColumnSchema wrapperColumnSchema = new ColumnSchema();
wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
- wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
DataType dataType = thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type);
if (DataTypes.isDecimal(dataType)) {
DecimalType decimalType = (DecimalType) dataType;
@@ -305,7 +304,6 @@ public abstract class AbstractDataFileFooterConverter {
wrapperColumnSchema.setEncodingList(encoders);
wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());
wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());
- wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
wrapperColumnSchema.setScale(externalColumnSchema.getScale());
wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
Map<String, String> properties = externalColumnSchema.getColumnProperties();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java
deleted file mode 100644
index c1f3f5e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMergerUtil.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.util;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-
-/**
- * Util class for merge activities of 2 loads.
- */
-public class CarbonMergerUtil {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CarbonMergerUtil.class.getName());
-
- public static int[] getCardinalityFromLevelMetadata(String path, String tableName) {
- int[] localCardinality = null;
- try {
- localCardinality = CarbonUtil.getCardinalityFromLevelMetadataFile(
- path + '/' + CarbonCommonConstants.LEVEL_METADATA_FILE + tableName + ".metadata");
- } catch (IOException e) {
- LOGGER.error("Error occurred :: " + e.getMessage());
- }
-
- return localCardinality;
- }
-
- /**
- * read from the first non-empty level metadata
- * @param paths paths
- * @param tableName table name
- * @return cardinality
- */
- public static int[] getCardinalityFromLevelMetadata(String[] paths, String tableName) {
- int[] localCardinality = null;
- for (String path : paths) {
- localCardinality = getCardinalityFromLevelMetadata(path, tableName);
- if (null != localCardinality) {
- break;
- }
- }
- return localCardinality;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 58de030..8fc648b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -20,9 +20,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.blocklet.BlockletEncodedColumnPage;
import org.apache.carbondata.core.datastore.blocklet.EncodedBlocklet;
import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
@@ -42,7 +40,6 @@ import org.apache.carbondata.format.ColumnSchema;
import org.apache.carbondata.format.CompressionCodec;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.DataChunk3;
-import org.apache.carbondata.format.Encoding;
import org.apache.carbondata.format.FileFooter3;
import org.apache.carbondata.format.FileHeader;
import org.apache.carbondata.format.IndexHeader;
@@ -54,6 +51,9 @@ import org.apache.carbondata.format.SegmentInfo;
*/
public class CarbonMetadataUtil {
+ private CarbonMetadataUtil() {
+ }
+
/**
* Below method prepares the file footer object for carbon data file version 3
*
@@ -222,28 +222,6 @@ public class CarbonMetadataUtil {
}
/**
- * @param blockIndex
- * @param encoding
- * @param columnSchemas
- * @param segmentProperties
- * @return return true if given encoding is present in column
- */
- private static boolean containsEncoding(int blockIndex, Encoding encoding,
- List<ColumnSchema> columnSchemas, SegmentProperties segmentProperties) {
- Set<Integer> dimOrdinals = segmentProperties.getDimensionOrdinalForBlock(blockIndex);
- // column groups will always have dictionary encoding
- if (dimOrdinals.size() > 1 && Encoding.DICTIONARY == encoding) {
- return true;
- }
- for (Integer dimOrdinal : dimOrdinals) {
- if (columnSchemas.get(dimOrdinal).encoders.contains(encoding)) {
- return true;
- }
- }
- return false;
- }
-
- /**
* Right now it is set to default values. We may use this in future
*/
public static ChunkCompressionMeta getSnappyChunkCompressionMeta() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 4df10ee..f7ace5e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -821,10 +821,6 @@ public final class CarbonProperties {
if (null != sessionPropertyValue) {
return sessionPropertyValue;
}
- //TODO temporary fix
- if ("carbon.leaf.node.size".equals(key)) {
- return "120000";
- }
return carbonProperties.getProperty(key);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 8afba76..e87e52c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -36,7 +36,6 @@ import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
@@ -246,26 +245,6 @@ public final class CarbonUtil {
}
/**
- * return ColumnGroupModel. check ColumnGroupModel for detail
- *
- * @param columnGroups : column groups
- * @return ColumnGroupModel model
- */
- public static ColumnGroupModel getColGroupModel(int[][] columnGroups) {
- int[] columnSplit = new int[columnGroups.length];
- int noOfColumnStore = columnSplit.length;
-
- for (int i = 0; i < columnGroups.length; i++) {
- columnSplit[i] = columnGroups[i].length;
- }
- ColumnGroupModel colGroupModel = new ColumnGroupModel();
- colGroupModel.setNoOfColumnStore(noOfColumnStore);
- colGroupModel.setColumnSplit(columnSplit);
- colGroupModel.setColumnGroup(columnGroups);
- return colGroupModel;
- }
-
- /**
* This method will be used to update the dimension cardinality
*
* @param dimCardinality
@@ -617,35 +596,6 @@ public final class CarbonUtil {
}
/**
- * Read level metadata file and return cardinality
- *
- * @param levelPath
- * @return
- * @throws IOException
- */
- public static int[] getCardinalityFromLevelMetadataFile(String levelPath) throws IOException {
- DataInputStream dataInputStream = null;
- int[] cardinality = null;
-
- try {
- if (FileFactory.isFileExist(levelPath, FileFactory.getFileType(levelPath))) {
- dataInputStream =
- FileFactory.getDataInputStream(levelPath, FileFactory.getFileType(levelPath));
-
- cardinality = new int[dataInputStream.readInt()];
-
- for (int i = 0; i < cardinality.length; i++) {
- cardinality[i] = dataInputStream.readInt();
- }
- }
- } finally {
- closeStreams(dataInputStream);
- }
-
- return cardinality;
- }
-
- /**
* From beeline if a delimeter is passed as \001, in code we get it as
* escaped string as \\001. So this method will unescape the slash again and
* convert it back t0 \001
@@ -1030,17 +980,11 @@ public final class CarbonUtil {
*/
public static int getNumberOfDimensionColumns(List<ColumnSchema> columnSchemaList) {
int numberOfDimensionColumns = 0;
- int previousColumnGroupId = -1;
ColumnSchema columnSchema = null;
for (int i = 0; i < columnSchemaList.size(); i++) {
columnSchema = columnSchemaList.get(i);
- if (columnSchema.isDimensionColumn() && columnSchema.isColumnar()) {
+ if (columnSchema.isDimensionColumn()) {
numberOfDimensionColumns++;
- } else if (columnSchema.isDimensionColumn()) {
- if (previousColumnGroupId != columnSchema.getColumnGroupId()) {
- previousColumnGroupId = columnSchema.getColumnGroupId();
- numberOfDimensionColumns++;
- }
} else {
break;
}
@@ -1142,20 +1086,14 @@ public final class CarbonUtil {
*/
public static boolean[] identifyDimensionType(List<CarbonDimension> tableDimensionList) {
List<Boolean> isDictionaryDimensions = new ArrayList<Boolean>();
- Set<Integer> processedColumnGroup = new HashSet<Integer>();
for (CarbonDimension carbonDimension : tableDimensionList) {
List<CarbonDimension> childs = carbonDimension.getListOfChildDimensions();
//assuming complex dimensions will always be atlast
if (null != childs && childs.size() > 0) {
break;
}
- if (carbonDimension.isColumnar() &&
- hasEncoding(carbonDimension.getEncoder(), Encoding.DICTIONARY)) {
+ if (hasEncoding(carbonDimension.getEncoder(), Encoding.DICTIONARY)) {
isDictionaryDimensions.add(true);
- } else if (!carbonDimension.isColumnar()) {
- if (processedColumnGroup.add(carbonDimension.columnGroupId())) {
- isDictionaryDimensions.add(true);
- }
} else {
isDictionaryDimensions.add(false);
}
@@ -1697,7 +1635,6 @@ public final class CarbonUtil {
return valueEncoderMeta;
}
-
/**
* Below method will be used to convert indexes in range
* Indexes=[0,1,2,3,4,5,6,7,8,9]
@@ -2195,7 +2132,6 @@ public final class CarbonUtil {
wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
- wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
DataType dataType = thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type);
if (DataTypes.isDecimal(dataType)) {
DecimalType decimalType = (DecimalType) dataType;
@@ -2211,7 +2147,6 @@ public final class CarbonUtil {
wrapperColumnSchema.setEncodingList(encoders);
wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());
wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());
- wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
wrapperColumnSchema.setScale(externalColumnSchema.getScale());
wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
index 07391dc..b20a336 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
@@ -126,13 +126,8 @@ public class DataFileFooterConverter2 extends AbstractDataFileFooterConverter {
ColumnSchema columnSchema = null;
for (int i = 0; i < columnSchemaList.size(); i++) {
columnSchema = columnSchemaList.get(i);
- if (columnSchema.isDimensionColumn() && columnSchema.isColumnar()) {
+ if (columnSchema.isDimensionColumn()) {
numberOfDimensionColumns++;
- } else if (columnSchema.isDimensionColumn()) {
- if (previousColumnGroupId != columnSchema.getColumnGroupId()) {
- previousColumnGroupId = columnSchema.getColumnGroupId();
- numberOfDimensionColumns++;
- }
} else {
break;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java b/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
deleted file mode 100644
index 2adfadc..0000000
--- a/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
+++ /dev/null
@@ -1,457 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.util;
-
-import java.util.BitSet;
-
-import org.apache.carbondata.core.datastore.page.EncodedTablePage;
-import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult;
-
-public class NodeHolder {
- private EncodedTablePage encodedData;
-
- /**
- * keyArray
- */
- private byte[][] keyArray;
-
- /**
- * dataArray
- */
- private byte[][] dataArray;
-
- /**
- * measureLenght
- */
- private int[] measureLenght;
-
- /**
- * startKey
- */
- private byte[] startKey;
-
- /**
- * endKey
- */
- private byte[] endKey;
-
- /**
- * entryCount
- */
- private int entryCount;
- /**
- * keyLenghts
- */
- private int[] keyLengths;
-
- /**
- * keyIndexBlockLenght
- */
- private int[] keyBlockIndexLength;
-
- /**
- * isSortedKeyBlock
- */
- private boolean[] isSortedKeyBlock;
-
- private byte[][] compressedIndex;
-
- private byte[][] compressedIndexMap;
-
- /**
- * dataIndexMap
- */
- private int[] dataIndexMapLength;
-
- /**
- * compressedDataIndex
- */
- private byte[][] compressedDataIndex;
-
- /**
- * column max data
- */
- private byte[][] dimensionColumnMaxData;
-
- /**
- * column min data
- */
- private byte[][] dimensionColumnMinData;
-
- private byte[][] measureColumnMaxData;
-
- private byte[][] measureColumnMinData;
-
- private SimpleStatsResult stats;
-
- /**
- * array of rleEncodingForDictDim flag to identify the rleEncodingForDictDim
- */
- private boolean[] rleEncodingForDictDim;
-
- /**
- * true if given index is colgroup block
- */
- private boolean[] colGrpBlocks;
-
- /**
- * bit set which will holds the measure
- * indexes which are null
- */
- private BitSet[] measureNullValueIndex;
-
- /**
- * total length of dimension values
- */
- private int totalDimensionArrayLength;
-
- /**
- * total length of all measure values
- */
- private int totalMeasureArrayLength;
-
- /**
- * data size this node holder is holding
- */
- private int holderSize;
-
- /**
- * to check all the pages to be
- * written, this will be used for v3 format
- */
- private boolean writeAll;
-
- /**
- * @return the keyArray
- */
- public byte[][] getKeyArray() {
- return keyArray;
- }
-
- /**
- * @param keyArray the keyArray to set
- */
- public void setKeyArray(byte[][] keyArray) {
- this.keyArray = keyArray;
- }
-
- /**
- * @return the dataArray
- */
- public byte[][] getDataArray() {
- return dataArray;
- }
-
- /**
- * @param dataArray the dataArray to set
- */
- public void setDataArray(byte[][] dataArray) {
- this.dataArray = dataArray;
- }
-
- /**
- * @return the measureLenght
- */
- public int[] getMeasureLenght() {
- return measureLenght;
- }
-
- /**
- * @param measureLenght the measureLenght to set
- */
- public void setMeasureLenght(int[] measureLenght) {
- this.measureLenght = measureLenght;
- }
-
- /**
- * @return the startKey
- */
- public byte[] getStartKey() {
- return startKey;
- }
-
- /**
- * @param startKey the startKey to set
- */
- public void setStartKey(byte[] startKey) {
- this.startKey = startKey;
- }
-
- /**
- * @return the endKey
- */
- public byte[] getEndKey() {
- return endKey;
- }
-
- /**
- * @param endKey the endKey to set
- */
- public void setEndKey(byte[] endKey) {
- this.endKey = endKey;
- }
-
- /**
- * @return the entryCount
- */
- public int getEntryCount() {
- return entryCount;
- }
-
- /**
- * @param entryCount the entryCount to set
- */
- public void setEntryCount(int entryCount) {
- this.entryCount = entryCount;
- }
-
- /**
- * @return the keyLenghts
- */
- public int[] getKeyLengths() {
- return keyLengths;
- }
-
- public void setKeyLengths(int[] keyLengths) {
- this.keyLengths = keyLengths;
- }
-
- /**
- * @return the keyBlockIndexLength
- */
- public int[] getKeyBlockIndexLength() {
- return keyBlockIndexLength;
- }
-
- /**
- * @param keyBlockIndexLength the keyBlockIndexLength to set
- */
- public void setKeyBlockIndexLength(int[] keyBlockIndexLength) {
- this.keyBlockIndexLength = keyBlockIndexLength;
- }
-
- /**
- * @return the isSortedKeyBlock
- */
- public boolean[] getIsSortedKeyBlock() {
- return isSortedKeyBlock;
- }
-
- /**
- * @param isSortedKeyBlock the isSortedKeyBlock to set
- */
- public void setIsSortedKeyBlock(boolean[] isSortedKeyBlock) {
- this.isSortedKeyBlock = isSortedKeyBlock;
- }
-
- /**
- * @return the compressedIndexex
- */
- public byte[][] getCompressedIndex() {
- return compressedIndex;
- }
-
- public void setCompressedIndex(byte[][] compressedIndex) {
- this.compressedIndex = compressedIndex;
- }
-
- /**
- * @return the compressedIndexMap
- */
- public byte[][] getCompressedIndexMap() {
- return compressedIndexMap;
- }
-
- /**
- * @param compressedIndexMap the compressedIndexMap to set
- */
- public void setCompressedIndexMap(byte[][] compressedIndexMap) {
- this.compressedIndexMap = compressedIndexMap;
- }
-
- /**
- * @return the compressedDataIndex
- */
- public byte[][] getCompressedDataIndex() {
- return compressedDataIndex;
- }
-
- /**
- * @param compressedDataIndex the compressedDataIndex to set
- */
- public void setCompressedDataIndex(byte[][] compressedDataIndex) {
- this.compressedDataIndex = compressedDataIndex;
- }
-
- /**
- * @return the dataIndexMapLength
- */
- public int[] getDataIndexMapLength() {
- return dataIndexMapLength;
- }
-
- /**
- * @param dataIndexMapLength the dataIndexMapLength to set
- */
- public void setDataIndexMapLength(int[] dataIndexMapLength) {
- this.dataIndexMapLength = dataIndexMapLength;
- }
-
- public byte[][] getDimensionColumnMaxData() {
- return this.dimensionColumnMaxData;
- }
-
- public void setDimensionColumnMaxData(byte[][] columnMaxData) {
- this.dimensionColumnMaxData = columnMaxData;
- }
-
- public byte[][] getDimensionColumnMinData() {
- return this.dimensionColumnMinData;
- }
-
- public void setDimensionColumnMinData(byte[][] columnMinData) {
- this.dimensionColumnMinData = columnMinData;
- }
-
- /**
- * returns array of rleEncodingForDictDim flag to identify the aag blocks
- *
- * @return
- */
- public boolean[] getRleEncodingForDictDim() {
- return rleEncodingForDictDim;
- }
-
- /**
- * set array of rleEncodingForDictDim flag to identify the rleEncodingForDictDim
- *
- * @param rleEncodingForDictDim
- */
- public void setRleEncodingForDictDim(boolean[] rleEncodingForDictDim) {
- this.rleEncodingForDictDim = rleEncodingForDictDim;
- }
-
- /**
- * @return
- */
- public boolean[] getColGrpBlocks() {
- return this.colGrpBlocks;
- }
-
- /**
- * @param colGrpBlock true if block is column group
- */
- public void setColGrpBlocks(boolean[] colGrpBlock) {
- this.colGrpBlocks = colGrpBlock;
- }
-
- /**
- * @return the measureNullValueIndex
- */
- public BitSet[] getMeasureNullValueIndex() {
- return measureNullValueIndex;
- }
-
- /**
- * @param measureNullValueIndex the measureNullValueIndex to set
- */
- public void setMeasureNullValueIndex(BitSet[] measureNullValueIndex) {
- this.measureNullValueIndex = measureNullValueIndex;
- }
-
- public int getTotalDimensionArrayLength() {
- return totalDimensionArrayLength;
- }
-
- public void setTotalDimensionArrayLength(int totalDimensionArrayLength) {
- this.totalDimensionArrayLength = totalDimensionArrayLength;
- }
-
- public int getTotalMeasureArrayLength() {
- return totalMeasureArrayLength;
- }
-
- public void setTotalMeasureArrayLength(int totalMeasureArrayLength) {
- this.totalMeasureArrayLength = totalMeasureArrayLength;
- }
-
- public byte[][] getMeasureColumnMaxData() {
- return measureColumnMaxData;
- }
-
- public void setMeasureColumnMaxData(byte[][] measureColumnMaxData) {
- this.measureColumnMaxData = measureColumnMaxData;
- }
-
- public byte[][] getMeasureColumnMinData() {
- return measureColumnMinData;
- }
-
- public void setMeasureColumnMinData(byte[][] measureColumnMinData) {
- this.measureColumnMinData = measureColumnMinData;
- }
-
- public int getHolderSize() {
- return holderSize;
- }
-
- public void setHolderSize(int holderSize) {
- this.holderSize = holderSize;
- }
-
- public void setWriteAll(boolean writeAll) {
- this.writeAll = writeAll;
- }
- public boolean isWriteAll() {
- return this.writeAll;
- }
-
- public SimpleStatsResult getStats() {
- return stats;
- }
-
- public void setMeasureStats(SimpleStatsResult stats) {
- this.stats = stats;
- }
-
- public static byte[][] getKeyArray(EncodedTablePage encodedTablePage) {
- int numDimensions = encodedTablePage.getNumDimensions();
- byte[][] keyArray = new byte[numDimensions][];
- for (int i = 0; i < numDimensions; i++) {
- keyArray[i] = encodedTablePage.getDimension(i).getEncodedData().array();
- }
- return keyArray;
- }
-
- public static byte[][] getDataArray(EncodedTablePage encodedTablePage) {
- int numMeasures = encodedTablePage.getNumMeasures();
- byte[][] dataArray = new byte[numMeasures][];
- for (int i = 0; i < numMeasures; i++) {
- dataArray[i] = encodedTablePage.getMeasure(i).getEncodedData().array();
- }
- return dataArray;
- }
-
- public void setEncodedData(EncodedTablePage encodedData) {
- this.encodedData = encodedData;
- }
-
- public EncodedTablePage getEncodedData() {
- return encodedData;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java b/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java
index bd2d4c2..16dacb2 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/TaskMetricsMap.java
@@ -130,10 +130,6 @@ public class TaskMetricsMap {
return sum;
}
- public void clear() {
- metricMap.clear();
- }
-
/**
* adds spawaned thread callback entry in metricmap using parentThreadId
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 14117f6..c88e248 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -155,13 +155,6 @@ public class CarbonTablePath {
}
/**
- * Return absolute path of sort index file
- */
- public static String getSortIndexFilePath(String tablePath, String columnId) {
- return getMetadataPath(tablePath) + File.separator + columnId + SORT_INDEX_EXT;
- }
-
- /**
* Return sortindex file path based on specified dictionary path
*/
public static String getExternalSortIndexFilePath(String dictionaryPath, String columnId) {
@@ -216,20 +209,6 @@ public class CarbonTablePath {
}
/**
- * Gets absolute path of data file
- *
- * @param segmentId unique partition identifier
- * @param filePartNo data file part number
- * @param factUpdateTimeStamp unique identifier to identify an update
- * @return absolute path of data file stored in carbon data format
- */
- public static String getCarbonDataFilePath(String tablePath, String segmentId, Integer filePartNo,
- Long taskNo, int batchNo, int bucketNumber, String factUpdateTimeStamp) {
- return getSegmentPath(tablePath, segmentId) + File.separator + getCarbonDataFileName(
- filePartNo, taskNo, bucketNumber, batchNo, factUpdateTimeStamp, segmentId);
- }
-
- /**
* Below method will be used to get the index file present in the segment folder
* based on task id
*
@@ -285,20 +264,6 @@ public class CarbonTablePath {
}
}
- public static String getCarbonIndexFilePath(String tablePath, String taskId, String segmentId,
- int batchNo, String bucketNumber, String timeStamp,
- ColumnarFormatVersion columnarFormatVersion) {
- switch (columnarFormatVersion) {
- case V1:
- case V2:
- return getCarbonIndexFilePath(tablePath, taskId, segmentId, bucketNumber);
- default:
- String segmentDir = getSegmentPath(tablePath, segmentId);
- return segmentDir + File.separator + getCarbonIndexFileName(Long.parseLong(taskId),
- Integer.parseInt(bucketNumber), batchNo, timeStamp, segmentId);
- }
- }
-
private static String getCarbonIndexFileName(String taskNo, int bucketNumber,
String factUpdatedtimeStamp, String segmentNo) {
if (bucketNumber == -1) {
@@ -359,13 +324,6 @@ public class CarbonTablePath {
// This partition is not used in any code logic, just keep backward compatibility
public static final String DEPRECATED_PATITION_ID = "0";
- /**
- * Return true if tablePath exists
- */
- public static boolean exists(String tablePath) {
- return FileFactory.getCarbonFile(tablePath, FileFactory.getFileType(tablePath)).exists();
- }
-
public static String getPartitionDir(String tablePath) {
return getFactDir(tablePath) + File.separator + PARTITION_PREFIX +
CarbonTablePath.DEPRECATED_PATITION_ID;
@@ -734,14 +692,6 @@ public class CarbonTablePath {
}
/**
- * Get the segment lock file according to table path and segment load name
- */
- public static String getSegmentLockFilePath(String tablePath, String loadName) {
- return getLockFilesDirPath(tablePath) + CarbonCommonConstants.FILE_SEPARATOR +
- addSegmentPrefix(loadName) + LockUsage.LOCK;
- }
-
- /**
* return true if this lock file is a segment lock file otherwise false.
*/
public static boolean isSegmentLockFilePath(String lockFileName) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/writer/ByteArrayHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/ByteArrayHolder.java b/core/src/main/java/org/apache/carbondata/core/writer/ByteArrayHolder.java
deleted file mode 100644
index 35c1f52..0000000
--- a/core/src/main/java/org/apache/carbondata/core/writer/ByteArrayHolder.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.writer;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class ByteArrayHolder implements Comparable<ByteArrayHolder> {
-
- /**
- * mdkey
- */
- private byte[] mdKey;
-
- /**
- * primary key
- */
- private int primaryKey;
-
- /**
- * @param mdKey
- * @param primaryKey
- */
- public ByteArrayHolder(byte[] mdKey, int primaryKey) {
- this.mdKey = mdKey;
- this.primaryKey = primaryKey;
- }
-
- @Override public int compareTo(ByteArrayHolder o) {
- return ByteUtil.compare(mdKey, o.mdKey);
- }
-
- @Override public boolean equals(Object obj) {
- // TODO Auto-generated method stub
- if (obj instanceof ByteArrayHolder) {
- if (0 == ByteUtil.compare(mdKey, ((ByteArrayHolder) obj).mdKey)) {
- return true;
- }
- }
- return false;
- }
-
- @Override public int hashCode() {
- int prime = 31;
- int result = prime * Arrays.hashCode(mdKey);
- result = result + prime * primaryKey;
- return result;
- }
-
- public byte[] getMdKey() {
- return mdKey;
- }
-
- public int getPrimaryKey() {
- return primaryKey;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java
index 3db1c4f..f0cc409 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java
@@ -25,12 +25,5 @@ import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails;
*/
public interface CarbonDeleteDeltaWriter {
- /**
- * write method that accepts comma seperated deleted records
- *
- * @param value delete record details
- * @throws IOException if an I/O error occurs
- */
- void write(String value) throws IOException;
void write(DeleteDeltaBlockDetails deleteBlockDetails) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
index 506d290..4cf3827 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
@@ -58,38 +58,6 @@ public class CarbonDeleteDeltaWriterImpl implements CarbonDeleteDeltaWriter {
}
/**
- * This method will write the deleted records data in to disk.
- *
- * @param value deleted records
- * @throws IOException if an I/O error occurs
- */
- @Override public void write(String value) throws IOException {
- BufferedWriter brWriter = null;
- try {
- FileFactory.createNewFile(filePath, fileType);
- dataOutStream = FileFactory.getDataOutputStream(filePath, fileType);
- brWriter = new BufferedWriter(new OutputStreamWriter(dataOutStream,
- CarbonCommonConstants.DEFAULT_CHARSET));
- brWriter.write(value);
- } catch (IOException ioe) {
- LOGGER.error("Error message: " + ioe.getLocalizedMessage());
- throw ioe;
- } finally {
- if (null != brWriter) {
- brWriter.flush();
- }
- if (null != dataOutStream) {
- dataOutStream.flush();
- }
- if (null != brWriter) {
- brWriter.close();
- }
-
- }
-
- }
-
- /**
* This method will write the deleted records data in the json format.
* @param deleteDeltaBlockDetails
* @throws IOException
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
index d7b1a0f..28f81f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
@@ -26,7 +26,6 @@ import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
import org.apache.carbondata.core.fileoperations.FileWriteOperation;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.thrift.TBase;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TCompactProtocol;
@@ -122,16 +121,6 @@ public class ThriftWriter {
}
/**
- * Write the offset to the file
- *
- * @param offset
- * @throws IOException
- */
- public void writeOffset(long offset) throws IOException {
- dataOutputStream.writeLong(offset);
- }
-
- /**
* Close the file stream.
*/
public void close() throws IOException {
@@ -151,13 +140,4 @@ public class ThriftWriter {
dataOutputStream = null;
}
}
-
- /**
- * Flush data to HDFS file
- */
- public void sync() throws IOException {
- if (dataOutputStream instanceof FSDataOutputStream) {
- ((FSDataOutputStream) dataOutputStream).hsync();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java b/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java
index 53349b8..5f9a05c 100644
--- a/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java
+++ b/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java
@@ -68,40 +68,6 @@ public class OperationListenerBus {
}
/**
- * Removes a given map from the event. This method is thread-safe and can be called
- * in any thread.
- *
- * @param eventType
- */
- public void removeEvent(String eventType) {
- eventMap.remove(eventType);
- }
-
- /**
- * Removes a given listener for an event. This method is thread-safe and can be called
- * in any thread.
- *
- * @param eventType
- * @param operationEventListener
- */
- public void removeListener(String eventType, OperationEventListener operationEventListener) {
- List<OperationEventListener> operationEventListeners = eventMap.get(eventType);
- if (null != operationEventListeners) {
- operationEventListeners.remove(operationEventListener);
- }
- }
-
- /**
- * Notify all registered listeners on occurrence of an event
- * Should be used for stateless events which cannot be mapped to a operation
- *
- * @param event
- */
- public void fireEvent(Event event) throws Exception {
- fireEvent(event, new OperationContext());
- }
-
- /**
* Notify all registered listeners on occurrence of an event
*
* @param event
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
index 7c48f37..deb1a29 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/CacheProviderTest.java
@@ -17,22 +17,16 @@
package org.apache.carbondata.core.cache;
-import java.io.IOException;
-import java.lang.reflect.Field;
-
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache;
import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
-import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -72,32 +66,4 @@ public class CacheProviderTest {
assertFalse(reverseDictionaryCache instanceof ForwardDictionaryCache);
cacheProvider.dropAllCache();
}
-
- /**
- * to test the driver and executor lru memory configuration
- *
- * @throws IOException
- * @throws NoSuchFieldException
- * @throws IllegalAccessException
- */
- @Test public void driverExecutorCacheConfTest()
- throws IOException, NoSuchFieldException, IllegalAccessException {
- // get cache provider instance
- CacheProvider cacheProvider = CacheProvider.getInstance();
- cacheProvider.dropAllCache();
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true");
- Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexStore> driverCache =
- cacheProvider.createCache(CacheType.DRIVER_BTREE);
- Field carbonLRUCacheField = SegmentTaskIndexStore.class.getDeclaredField("lruCache");
- carbonLRUCacheField.setAccessible(true);
- CarbonLRUCache carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(driverCache);
- Field lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize");
- lruCacheMemorySizeField.setAccessible(true);
- long lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache);
- String driverCacheSize = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE);
- assertEquals(1024 * 1024 * Integer.parseInt(driverCacheSize), lruCacheMemorySize);
- // drop cache
- cacheProvider.dropAllCache();
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
index 7d829b9..b572a36 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
@@ -87,9 +87,9 @@ public class AbstractDictionaryCacheTest {
* prepare the dataset required for running test cases
*/
protected void prepareDataSet() {
- dataSet1 = Arrays.asList(new String[] { "a", "b", "c" });
- dataSet2 = Arrays.asList(new String[] { "d", "e", "f" });
- dataSet3 = Arrays.asList(new String[] { "b", "c", "a", "d" });
+ dataSet1 = Arrays.asList("a", "b", "c");
+ dataSet2 = Arrays.asList("d", "e", "f");
+ dataSet3 = Arrays.asList("b", "c", "a", "d");
}
/**
[2/8] carbondata git commit: [CARBONDATA-2720] Remove dead code
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/events/AlterTableEvents.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/events/AlterTableEvents.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/events/AlterTableEvents.scala
index 55fa2bd..37abd73 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/events/AlterTableEvents.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/events/AlterTableEvents.scala
@@ -193,26 +193,13 @@ case class AlterTableCompactionPostStatusUpdateEvent(sparkSession: SparkSession,
/**
* Compaction Event for handling clean up in case of any compaction failure and abort the
- * operation, lister has to implement this event to handle failure scenarios
- *
- * @param carbonTable
- * @param carbonMergerMapping
- * @param mergedLoadName
- */
-case class AlterTableCompactionAbortEvent(sparkSession: SparkSession,
- carbonTable: CarbonTable,
- carbonMergerMapping: CarbonMergerMapping,
- mergedLoadName: String) extends Event with AlterTableCompactionEventInfo
-
-
-/**
- * Compaction Event for handling exception in compaction
+ * * operation, lister has to implement this event to handle failure scenarios
*
* @param sparkSession
* @param carbonTable
* @param alterTableModel
*/
-case class AlterTableCompactionExceptionEvent(sparkSession: SparkSession,
+case class AlterTableCompactionAbortEvent(sparkSession: SparkSession,
carbonTable: CarbonTable,
alterTableModel: AlterTableModel) extends Event with AlterTableCompactionEventInfo
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/events/CleanFilesEvents.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/events/CleanFilesEvents.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/events/CleanFilesEvents.scala
index 1a9c5f6..b7e9c20 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/events/CleanFilesEvents.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/events/CleanFilesEvents.scala
@@ -36,11 +36,3 @@ case class CleanFilesPreEvent(carbonTable: CarbonTable, sparkSession: SparkSessi
*/
case class CleanFilesPostEvent(carbonTable: CarbonTable, sparkSession: SparkSession)
extends Event with CleanFilesEventInfo
-
-/**
- *
- * @param carbonTable
- * @param sparkSession
- */
-case class CleanFilesAbortEvent(carbonTable: CarbonTable, sparkSession: SparkSession)
- extends Event with CleanFilesEventInfo
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
index 5f23f77..9724fa8 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
@@ -31,11 +31,6 @@ class CarbonOption(options: Map[String, String]) {
def partitionCount: String = options.getOrElse("partitionCount", "1")
- def partitionClass: String = {
- options.getOrElse("partitionClass",
- "org.apache.carbondata.processing.partition.impl.SampleDataPartitionerImpl")
- }
-
def tempCSV: Boolean = options.getOrElse("tempCSV", "false").toBoolean
def compress: Boolean = options.getOrElse("compress", "false").toBoolean
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index f443214..73ed769 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -254,8 +254,7 @@ object DataLoadProcessorStepOnSpark {
while (rows.hasNext) {
if (rowsNotExist) {
rowsNotExist = false
- dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(dataHandlerModel,
- CarbonFactHandlerFactory.FactHandlerType.COLUMNAR)
+ dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(dataHandlerModel)
dataHandler.initialise()
}
val row = dataWriter.processRow(rows.next(), dataHandler)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index cf22b3d..2ec8b9c 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -68,9 +68,6 @@ trait GenericParser {
def parseString(input: String): Unit
}
-case class DictionaryStats(distinctValues: java.util.List[String],
- dictWriteTime: Long, sortIndexWriteTime: Long)
-
case class PrimitiveParser(dimension: CarbonDimension,
setOpt: Option[mutable.HashSet[String]]) extends GenericParser {
val (hasDictEncoding, set: mutable.HashSet[String]) = setOpt match {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 401ba29..df3072b 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -47,15 +47,13 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.mutate.UpdateVO
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
import org.apache.carbondata.core.statusmanager.{FileFormat, SegmentUpdateStatusManager}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, DataTypeUtil}
-import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.hadoop.api.{CarbonInputFormat, CarbonTableInputFormat}
import org.apache.carbondata.hadoop.util.{CarbonInputFormatUtil, CarbonInputSplitTaskInfo}
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.merger._
-import org.apache.carbondata.processing.splits.TableSplit
import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil, CarbonLoaderUtil}
import org.apache.carbondata.spark.MergeResult
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, SparkDataTypeConverterImpl, Util}
@@ -517,16 +515,3 @@ class CarbonMergerRDD[K, V](
theSplit.split.value.getLocations.filter(_ != "localhost")
}
}
-
-
-class CarbonLoadPartition(rddId: Int, val idx: Int, @transient val tableSplit: TableSplit)
- extends Partition {
-
- override val index: Int = idx
- val serializableHadoopSplit = new SerializableWritable[TableSplit](tableSplit)
-
- override def hashCode(): Int = 41 * (41 + rddId) + idx
-}
-
-case class SplitTaskInfo (splits: List[CarbonInputSplit]) extends CarbonInputSplit{
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 80b2d12..6b136bc 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -50,7 +50,6 @@ import org.apache.carbondata.processing.loading.{DataLoadExecutor, FailureCauses
import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat, CSVRecordReaderIterator}
import org.apache.carbondata.processing.loading.exception.NoRetryException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
-import org.apache.carbondata.processing.splits.TableSplit
import org.apache.carbondata.processing.util.CarbonQueryUtil
import org.apache.carbondata.spark.DataLoadResult
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, Util}
@@ -103,21 +102,6 @@ class CarbonNodePartition(rddId: Int, val idx: Int, host: String,
override def hashCode(): Int = 41 * (41 + rddId) + idx
}
-/**
- * This partition class use to split by TableSplit
- *
- */
-class CarbonTableSplitPartition(rddId: Int, val idx: Int, @transient val tableSplit: TableSplit,
- val blocksDetails: Array[BlockDetails])
- extends Partition {
-
- override val index: Int = idx
- val serializableHadoopSplit = new SerializableWritable[TableSplit](tableSplit)
- val partitionBlocksDetail = blocksDetails
-
- override def hashCode(): Int = 41 * (41 + rddId) + idx
-}
-
class SparkPartitionLoader(model: CarbonLoadModel,
splitIndex: Long,
storePath: String,
@@ -140,7 +124,6 @@ class SparkPartitionLoader(model: CarbonLoadModel,
CarbonProperties.getInstance().addProperty("is.int.based.indexer", "true")
CarbonProperties.getInstance().addProperty("aggregate.columnar.keyblock", "true")
CarbonProperties.getInstance().addProperty("is.compressed.keyblock", "false")
- CarbonProperties.getInstance().addProperty("carbon.leaf.node.size", "120000")
// this property is used to determine whether temp location for carbon is inside
// container temp dir or is yarn application directory.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 30c1874..28cd7ef 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -88,23 +88,6 @@ object CarbonScalaUtil {
}
}
- def convertSparkToCarbonSchemaDataType(dataType: String): String = {
- dataType match {
- case CarbonCommonConstants.STRING_TYPE => CarbonCommonConstants.STRING
- case CarbonCommonConstants.INTEGER_TYPE => CarbonCommonConstants.INTEGER
- case CarbonCommonConstants.BYTE_TYPE => CarbonCommonConstants.INTEGER
- case CarbonCommonConstants.SHORT_TYPE => CarbonCommonConstants.SHORT
- case CarbonCommonConstants.LONG_TYPE => CarbonCommonConstants.NUMERIC
- case CarbonCommonConstants.DOUBLE_TYPE => CarbonCommonConstants.NUMERIC
- case CarbonCommonConstants.FLOAT_TYPE => CarbonCommonConstants.NUMERIC
- case CarbonCommonConstants.DECIMAL_TYPE => CarbonCommonConstants.NUMERIC
- case CarbonCommonConstants.DATE_TYPE => CarbonCommonConstants.STRING
- case CarbonCommonConstants.BOOLEAN_TYPE => CarbonCommonConstants.STRING
- case CarbonCommonConstants.TIMESTAMP_TYPE => CarbonCommonConstants.TIMESTAMP
- case anyType => anyType
- }
- }
-
def convertCarbonToSparkDataType(dataType: CarbonDataType): types.DataType = {
if (CarbonDataTypes.isDecimal(dataType)) {
DecimalType(dataType.asInstanceOf[CarbonDecimalType].getPrecision,
@@ -361,77 +344,6 @@ object CarbonScalaUtil {
}
/**
- * This method will validate a column for its data type and check whether the column data type
- * can be modified and update if conditions are met
- *
- * @param dataTypeInfo
- * @param carbonColumn
- */
- def validateColumnDataType(dataTypeInfo: DataTypeInfo, carbonColumn: CarbonColumn): Unit = {
- carbonColumn.getDataType.getName match {
- case "INT" =>
- if (!dataTypeInfo.dataType.equals("bigint") && !dataTypeInfo.dataType.equals("long")) {
- sys
- .error(s"Given column ${ carbonColumn.getColName } with data type ${
- carbonColumn
- .getDataType.getName
- } cannot be modified. Int can only be changed to bigInt or long")
- }
- case "DECIMAL" =>
- if (!dataTypeInfo.dataType.equals("decimal")) {
- sys
- .error(s"Given column ${ carbonColumn.getColName } with data type ${
- carbonColumn.getDataType.getName
- } cannot be modified. Decimal can be only be changed to Decimal of higher precision")
- }
- if (dataTypeInfo.precision <= carbonColumn.getColumnSchema.getPrecision) {
- sys
- .error(s"Given column ${
- carbonColumn
- .getColName
- } cannot be modified. Specified precision value ${
- dataTypeInfo
- .precision
- } should be greater than current precision value ${
- carbonColumn.getColumnSchema
- .getPrecision
- }")
- } else if (dataTypeInfo.scale < carbonColumn.getColumnSchema.getScale) {
- sys
- .error(s"Given column ${
- carbonColumn
- .getColName
- } cannot be modified. Specified scale value ${
- dataTypeInfo
- .scale
- } should be greater or equal to current scale value ${
- carbonColumn.getColumnSchema
- .getScale
- }")
- } else {
- // difference of precision and scale specified by user should not be less than the
- // difference of already existing precision and scale else it will result in data loss
- val carbonColumnPrecisionScaleDiff = carbonColumn.getColumnSchema.getPrecision -
- carbonColumn.getColumnSchema.getScale
- val dataInfoPrecisionScaleDiff = dataTypeInfo.precision - dataTypeInfo.scale
- if (dataInfoPrecisionScaleDiff < carbonColumnPrecisionScaleDiff) {
- sys
- .error(s"Given column ${
- carbonColumn
- .getColName
- } cannot be modified. Specified precision and scale values will lead to data loss")
- }
- }
- case _ =>
- sys
- .error(s"Given column ${ carbonColumn.getColName } with data type ${
- carbonColumn
- .getDataType.getName
- } cannot be modified. Only Int and Decimal data types are allowed for modification")
- }
- }
-
- /**
* returns all fields except tupleId field as it is not required in the value
*
* @param fields
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 2f08d07..39530f4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -65,77 +65,6 @@ object CommonUtil {
val FIXED_DECIMAL = """decimal\(\s*(\d+)\s*,\s*(\-?\d+)\s*\)""".r
val FIXED_DECIMALTYPE = """decimaltype\(\s*(\d+)\s*,\s*(\-?\d+)\s*\)""".r
- def validateColumnGroup(colGroup: String, noDictionaryDims: Seq[String],
- msrs: Seq[Field], retrievedColGrps: Seq[String], dims: Seq[Field]) {
- val colGrpCols = colGroup.split(',').map(_.trim)
- colGrpCols.foreach { x =>
- // if column is no dictionary
- if (noDictionaryDims.contains(x)) {
- throw new MalformedCarbonCommandException(
- "Column group is not supported for no dictionary columns:" + x)
- } else if (msrs.exists(msr => msr.column.equals(x))) {
- // if column is measure
- throw new MalformedCarbonCommandException("Column group is not supported for measures:" + x)
- } else if (foundIndExistingColGrp(x)) {
- throw new MalformedCarbonCommandException("Column is available in other column group:" + x)
- } else if (isComplex(x, dims)) {
- throw new MalformedCarbonCommandException(
- "Column group doesn't support Complex column:" + x)
- } else if (isTimeStampColumn(x, dims)) {
- throw new MalformedCarbonCommandException(
- "Column group doesn't support Timestamp datatype:" + x)
- }// if invalid column is
- else if (!dims.exists(dim => dim.column.equalsIgnoreCase(x))) {
- // present
- throw new MalformedCarbonCommandException(
- "column in column group is not a valid column: " + x
- )
- }
- }
- // check if given column is present in other groups
- def foundIndExistingColGrp(colName: String): Boolean = {
- retrievedColGrps.foreach { colGrp =>
- if (colGrp.split(",").contains(colName)) {
- return true
- }
- }
- false
- }
-
- }
-
-
- def isTimeStampColumn(colName: String, dims: Seq[Field]): Boolean = {
- dims.foreach { dim =>
- if (dim.column.equalsIgnoreCase(colName)) {
- if (dim.dataType.isDefined && null != dim.dataType.get &&
- "timestamp".equalsIgnoreCase(dim.dataType.get)) {
- return true
- }
- }
- }
- false
- }
-
- def isComplex(colName: String, dims: Seq[Field]): Boolean = {
- dims.foreach { x =>
- if (x.children.isDefined && null != x.children.get && x.children.get.nonEmpty) {
- val children = x.children.get
- if (x.column.equals(colName)) {
- return true
- } else {
- children.foreach { child =>
- val fieldName = x.column + "." + child.column
- if (fieldName.equalsIgnoreCase(colName)) {
- return true
- }
- }
- }
- }
- }
- false
- }
-
def getColumnProperties(column: String,
tableProperties: Map[String, String]): Option[util.List[ColumnProperty]] = {
val fieldProps = new util.ArrayList[ColumnProperty]()
@@ -438,38 +367,6 @@ object CommonUtil {
}
/**
- * @param colGrps
- * @param dims
- * @return columns of column groups in schema order
- */
- def arrangeColGrpsInSchemaOrder(colGrps: Seq[String], dims: Seq[Field]): Seq[String] = {
- def sortByIndex(colGrp1: String, colGrp2: String) = {
- val firstCol1 = colGrp1.split(",")(0)
- val firstCol2 = colGrp2.split(",")(0)
- val dimIndex1: Int = getDimIndex(firstCol1, dims)
- val dimIndex2: Int = getDimIndex(firstCol2, dims)
- dimIndex1 < dimIndex2
- }
- val sortedColGroups: Seq[String] = colGrps.sortWith(sortByIndex)
- sortedColGroups
- }
-
- /**
- * @param colName
- * @param dims
- * @return return index for given column in dims
- */
- def getDimIndex(colName: String, dims: Seq[Field]): Int = {
- var index: Int = -1
- dims.zipWithIndex.foreach { h =>
- if (h._1.column.equalsIgnoreCase(colName)) {
- index = h._2.toInt
- }
- }
- index
- }
-
- /**
* validate table level properties for compaction
*
* @param tableProperties
@@ -692,58 +589,6 @@ object CommonUtil {
}
}
- def validateMaxColumns(csvHeaders: Array[String], maxColumns: String): Int = {
- /*
- User configures both csvheadercolumns, maxcolumns,
- if csvheadercolumns >= maxcolumns, give error
- if maxcolumns > threashold, give error
- User configures csvheadercolumns
- if csvheadercolumns >= maxcolumns(default) then maxcolumns = csvheadercolumns+1
- if csvheadercolumns >= threashold, give error
- User configures nothing
- if csvheadercolumns >= maxcolumns(default) then maxcolumns = csvheadercolumns+1
- if csvheadercolumns >= threashold, give error
- */
- val columnCountInSchema = csvHeaders.length
- var maxNumberOfColumnsForParsing = 0
- val maxColumnsInt = getMaxColumnValue(maxColumns)
- if (maxColumnsInt != null) {
- if (columnCountInSchema >= maxColumnsInt) {
- CarbonException.analysisException(
- s"csv headers should be less than the max columns: $maxColumnsInt")
- } else if (maxColumnsInt > CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING) {
- CarbonException.analysisException(
- s"max columns cannot be greater than the threshold value: " +
- s"${CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING}")
- } else {
- maxNumberOfColumnsForParsing = maxColumnsInt
- }
- } else if (columnCountInSchema >= CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING) {
- CarbonException.analysisException(
- s"csv header columns should be less than max threashold: " +
- s"${CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING}")
- } else if (columnCountInSchema >= CSVInputFormat.DEFAULT_MAX_NUMBER_OF_COLUMNS_FOR_PARSING) {
- maxNumberOfColumnsForParsing = columnCountInSchema + 1
- } else {
- maxNumberOfColumnsForParsing = CSVInputFormat.DEFAULT_MAX_NUMBER_OF_COLUMNS_FOR_PARSING
- }
- maxNumberOfColumnsForParsing
- }
-
- private def getMaxColumnValue(maxColumn: String): Integer = {
- if (maxColumn != null) {
- try {
- maxColumn.toInt
- } catch {
- case e: Exception =>
- LOGGER.error(s"Invalid value for max column in load options ${ e.getMessage }")
- null
- }
- } else {
- null
- }
- }
-
def getPartitionInfo(columnName: String, partitionType: PartitionType,
partitionInfo: PartitionInfo): Seq[Row] = {
var result = Seq.newBuilder[Row]
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 6cd28c0..c8af869 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -82,28 +82,6 @@ object DataTypeConverterUtil {
}
}
- def convertToString(dataType: DataType): String = {
- if (DataTypes.isDecimal(dataType)) {
- "decimal"
- } else if (DataTypes.isArrayType(dataType)) {
- "array"
- } else if (DataTypes.isStructType(dataType)) {
- "struct"
- } else {
- dataType match {
- case DataTypes.BOOLEAN => "boolean"
- case DataTypes.STRING => "string"
- case DataTypes.SHORT => "smallint"
- case DataTypes.INT => "int"
- case DataTypes.LONG => "bigint"
- case DataTypes.DOUBLE => "double"
- case DataTypes.FLOAT => "double"
- case DataTypes.TIMESTAMP => "timestamp"
- case DataTypes.DATE => "date"
- }
- }
- }
-
/**
* convert from wrapper to external data type
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 5129e59..1bb3912 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -159,78 +159,6 @@ object GlobalDictionaryUtil {
dimensionsWithDict.toArray
}
- /**
- * invoke CarbonDictionaryWriter to write dictionary to file.
- *
- * @param model instance of DictionaryLoadModel
- * @param columnIndex the index of current column in column list
- * @param iter distinct value list of dictionary
- */
- def writeGlobalDictionaryToFile(model: DictionaryLoadModel,
- columnIndex: Int,
- iter: Iterator[String]): Unit = {
- val dictService = CarbonCommonFactory.getDictionaryService
- val dictionaryColumnUniqueIdentifier: DictionaryColumnUniqueIdentifier = new
- DictionaryColumnUniqueIdentifier(
- model.table,
- model.columnIdentifier(columnIndex),
- model.columnIdentifier(columnIndex).getDataType)
- val writer: CarbonDictionaryWriter = dictService
- .getDictionaryWriter(dictionaryColumnUniqueIdentifier)
- try {
- while (iter.hasNext) {
- writer.write(iter.next)
- }
- } finally {
- writer.close()
- }
- }
-
- /**
- * read global dictionary from cache
- */
- def readGlobalDictionaryFromCache(model: DictionaryLoadModel): HashMap[String, Dictionary] = {
- val dictMap = new HashMap[String, Dictionary]
- model.primDimensions.zipWithIndex.filter(f => model.dictFileExists(f._2)).foreach { m =>
- val dict = CarbonLoaderUtil.getDictionary(model.table,
- m._1.getColumnIdentifier, m._1.getDataType
- )
- dictMap.put(m._1.getColumnId, dict)
- }
- dictMap
- }
-
- /**
- * invoke CarbonDictionaryReader to read dictionary from files.
- *
- * @param model carbon dictionary load model
- */
- def readGlobalDictionaryFromFile(model: DictionaryLoadModel): HashMap[String, HashSet[String]] = {
- val dictMap = new HashMap[String, HashSet[String]]
- val dictService = CarbonCommonFactory.getDictionaryService
- for (i <- model.primDimensions.indices) {
- val dictionaryColumnUniqueIdentifier: DictionaryColumnUniqueIdentifier = new
- DictionaryColumnUniqueIdentifier(
- model.table,
- model.columnIdentifier(i),
- model.columnIdentifier(i).getDataType)
- val set = new HashSet[String]
- if (model.dictFileExists(i)) {
- val reader: CarbonDictionaryReader = dictService.getDictionaryReader(
- dictionaryColumnUniqueIdentifier)
- val values = reader.read
- if (values != null) {
- for (j <- 0 until values.size) {
- set.add(new String(values.get(j),
- Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)))
- }
- }
- }
- dictMap.put(model.primDimensions(i).getColumnId, set)
- }
- dictMap
- }
-
def generateParserForChildrenDimension(dim: CarbonDimension,
format: DataFormat,
mapColumnValuesWithId:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index 840b03f..cc8a28e 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -100,13 +100,8 @@ object StreamSinkFactory {
val operationContext = new OperationContext
val loadTablePreExecutionEvent = new LoadTablePreExecutionEvent(
carbonTable.getCarbonTableIdentifier,
- carbonLoadModel,
- carbonLoadModel.getFactFilePath,
- false,
- parameters.asJava,
- parameters.asJava,
- false
- )
+ carbonLoadModel
+ )
OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext)
// prepare the stream segment
val segmentId = getStreamSegmentId(carbonTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/spark/rdd/UpdateCoalescedRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/rdd/UpdateCoalescedRDD.scala b/integration/spark-common/src/main/scala/org/apache/spark/rdd/UpdateCoalescedRDD.scala
deleted file mode 100644
index bcca7ed..0000000
--- a/integration/spark-common/src/main/scala/org/apache/spark/rdd/UpdateCoalescedRDD.scala
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.rdd
-
-import scala.reflect.ClassTag
-
-import org.apache.spark._
-
-import org.apache.carbondata.spark.rdd.CarbonRDD
-
-
-// This RDD distributes previous RDD data based on number of nodes. i.e., one partition for one node
-
-class UpdateCoalescedRDD[T: ClassTag](
- @transient var prev: RDD[T],
- nodeList: Array[String])
- extends CarbonRDD[T](prev.context, Nil, prev.sparkContext.hadoopConfiguration) {
-
- override def getPartitions: Array[Partition] = {
- new DataLoadPartitionCoalescer(prev, nodeList).run
- }
-
- override def internalCompute(split: Partition,
- context: TaskContext): Iterator[T] = {
- // This iterator combines data from all the parent partitions
- new Iterator[T] {
- val parentPartitionIter = split.asInstanceOf[CoalescedRDDPartition].parents.iterator
- var currentDataIter: Iterator[T] = _
- val prevRdd = firstParent[T]
-
- def hasNext: Boolean = {
- while (currentDataIter == null ||
- !currentDataIter.hasNext &&
- parentPartitionIter.hasNext) {
- val currentPartition = parentPartitionIter.next()
- currentDataIter = prevRdd.compute(currentPartition, context)
- }
- if (currentDataIter == null) {
- false
- } else {
- currentDataIter.hasNext
- }
- }
-
- def next: T = {
- currentDataIter.next()
- }
- }
- }
-
- override def getDependencies: Seq[Dependency[_]] = {
- Seq(new NarrowDependency(prev) {
- def getParents(id: Int): Seq[Int] = {
- partitions(id).asInstanceOf[CoalescedRDDPartition].parentsIndices
- }
- })
- }
-
- override def clearDependencies() {
- super.clearDependencies()
- prev = null
- }
-
- /**
- * Returns the preferred machine for the partition. If split is of type CoalescedRDDPartition,
- * then the preferred machine will be one which most parent splits prefer too.
- *
- * @param partition
- * @return the machine most preferred by split
- */
- override def getPreferredLocations(partition: Partition): Seq[String] = {
- partition.asInstanceOf[CoalescedRDDPartition].preferredLocation.toSeq
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index d705fc9..0a0b49f 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -289,13 +289,6 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
// column properties
val colProps = extractColumnProperties(fields, tableProperties)
- // get column groups configuration from table properties.
- val groupCols: Seq[String] = updateColumnGroupsInField(tableProperties,
- noDictionaryDims, msrs, dims)
- if (groupCols != null) {
- throw new MalformedCarbonCommandException(
- s"${CarbonCommonConstants.COLUMN_GROUPS} is deprecated")
- }
// validate the local dictionary property if defined
if (tableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE).isDefined) {
@@ -409,7 +402,6 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
Option(varcharColumns),
Option(noDictionaryDims),
Option(noInvertedIdxCols),
- groupCols,
Some(colProps),
bucketFields: Option[BucketFields],
partitionInfo,
@@ -488,74 +480,6 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
/**
- * Extract the column groups configuration from table properties.
- * Based on this Row groups of fields will be determined.
- *
- * @param tableProperties
- * @return
- */
- protected def updateColumnGroupsInField(tableProperties: mutable.Map[String, String],
- noDictionaryDims: Seq[String],
- msrs: Seq[Field],
- dims: Seq[Field]): Seq[String] = {
- if (tableProperties.get(CarbonCommonConstants.COLUMN_GROUPS).isDefined) {
-
- var splittedColGrps: Seq[String] = Seq[String]()
- val nonSplitCols: String = tableProperties.get(CarbonCommonConstants.COLUMN_GROUPS).get
-
- // row groups will be specified in table properties like -> "(col1,col2),(col3,col4)"
- // here first splitting the value by () . so that the above will be splitted into 2 strings.
- // [col1,col2] [col3,col4]
- val m: Matcher = Pattern.compile("\\(([^)]+)\\)").matcher(nonSplitCols)
- while (m.find()) {
- val oneGroup: String = m.group(1)
- CommonUtil.validateColumnGroup(oneGroup, noDictionaryDims, msrs, splittedColGrps, dims)
- val arrangedColGrp = rearrangedColumnGroup(oneGroup, dims)
- splittedColGrps :+= arrangedColGrp
- }
- // This will be furthur handled.
- CommonUtil.arrangeColGrpsInSchemaOrder(splittedColGrps, dims)
- } else {
- null
- }
- }
-
- def rearrangedColumnGroup(colGroup: String, dims: Seq[Field]): String = {
- // if columns in column group is not in schema order than arrange it in schema order
- var colGrpFieldIndx: Seq[Int] = Seq[Int]()
- colGroup.split(',').map(_.trim).foreach { x =>
- dims.zipWithIndex.foreach { dim =>
- if (dim._1.column.equalsIgnoreCase(x)) {
- colGrpFieldIndx :+= dim._2
- }
- }
- }
- // sort it
- colGrpFieldIndx = colGrpFieldIndx.sorted
- // check if columns in column group is in schema order
- if (!checkIfInSequence(colGrpFieldIndx)) {
- throw new MalformedCarbonCommandException("Invalid column group:" + colGroup)
- }
- def checkIfInSequence(colGrpFieldIndx: Seq[Int]): Boolean = {
- for (i <- 0 until (colGrpFieldIndx.length - 1)) {
- if ((colGrpFieldIndx(i + 1) - colGrpFieldIndx(i)) != 1) {
- throw new MalformedCarbonCommandException(
- "Invalid column group,column in group should be contiguous as per schema.")
- }
- }
- true
- }
- val colGrpNames: StringBuilder = StringBuilder.newBuilder
- for (i <- colGrpFieldIndx.indices) {
- colGrpNames.append(dims(colGrpFieldIndx(i)).column)
- if (i < (colGrpFieldIndx.length - 1)) {
- colGrpNames.append(",")
- }
- }
- colGrpNames.toString()
- }
-
- /**
* @param partitionCols
* @param tableProperties
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index e50a8fd..a61f94f 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -63,7 +63,6 @@ case class TableModel(
varcharCols: Option[Seq[String]],
highcardinalitydims: Option[Seq[String]],
noInvertedIdxCols: Option[Seq[String]],
- columnGroups: Seq[String],
colProps: Option[util.Map[String, util.List[ColumnProperty]]] = None,
bucketFields: Option[BucketFields],
partitionInfo: Option[PartitionInfo],
@@ -98,9 +97,6 @@ case class ColumnProperty(key: String, value: String)
case class ComplexField(complexType: String, primitiveField: Option[Field],
complexField: Option[ComplexField])
-case class Partitioner(partitionClass: String, partitionColumn: Array[String], partitionCount: Int,
- nodeList: Array[String])
-
case class PartitionerField(partitionColumn: String, dataType: Option[String],
columnComment: String)
@@ -512,7 +508,6 @@ object TableNewProcessor {
val columnUniqueId = colUniqueIdGenerator.generateUniqueId(columnSchema)
columnSchema.setColumnUniqueId(columnUniqueId)
columnSchema.setColumnReferenceId(columnUniqueId)
- columnSchema.setColumnar(true)
columnSchema.setDimensionColumn(isDimensionCol)
columnSchema.setPrecision(precision)
columnSchema.setScale(scale)
@@ -761,10 +756,6 @@ class TableNewProcessor(cm: TableModel) {
val highCardinalityDims = cm.highcardinalitydims.getOrElse(Seq())
- checkColGroupsValidity(cm.columnGroups, allColumns, highCardinalityDims)
-
- updateColumnGroupsInFields(cm.columnGroups, allColumns)
-
// Setting the boolean value of useInvertedIndex in column schema, if Paranet table is defined
// Encoding is already decided above
if (!cm.parentTable.isDefined) {
@@ -925,25 +916,4 @@ class TableNewProcessor(cm: TableModel) {
})
}
}
-
- // For updating the col group details for fields.
- private def updateColumnGroupsInFields(colGrps: Seq[String], allCols: Seq[ColumnSchema]): Unit = {
- if (null != colGrps) {
- var colGroupId = -1
- colGrps.foreach(columngroup => {
- colGroupId += 1
- val rowCols = columngroup.split(",")
- rowCols.foreach(row => {
-
- allCols.foreach(eachCol => {
-
- if (eachCol.getColumnName.equalsIgnoreCase(row.trim)) {
- eachCol.setColumnGroup(colGroupId)
- eachCol.setColumnar(false)
- }
- })
- })
- })
- }
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index 44f96bd..ffaac86 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -113,12 +113,7 @@ class CarbonAppendableStreamSink(
// in case of streaming options and optionsFinal can be same
val loadTablePreExecutionEvent = new LoadTablePreExecutionEvent(
carbonTable.getCarbonTableIdentifier,
- carbonLoadModel,
- carbonLoadModel.getFactFilePath,
- false,
- parameters.asJava,
- parameters.asJava,
- false)
+ carbonLoadModel)
OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext)
checkOrHandOffSegment()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index 9b9e71d..24af8ec 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -31,6 +31,7 @@
<properties>
<dev.path>${basedir}/../../dev</dev.path>
+ <jacoco.append>true</jacoco.append>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
index 0642e01..cb5a1b1 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.exceptions.MetadataProcessException;
import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
-import org.apache.carbondata.core.datamap.DataMapCatalog;
import org.apache.carbondata.core.datamap.DataMapProvider;
import org.apache.carbondata.core.datamap.DataMapRegistry;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
@@ -81,11 +80,6 @@ public class IndexDataMapProvider extends DataMapProvider {
}
@Override
- public void initData() {
- // Nothing is needed to do by default
- }
-
- @Override
public void cleanMeta() throws IOException {
if (getMainTable() == null) {
throw new UnsupportedOperationException("Table need to be specified in index datamaps");
@@ -108,11 +102,6 @@ public class IndexDataMapProvider extends DataMapProvider {
IndexDataMapRebuildRDD.rebuildDataMap(sparkSession, getMainTable(), getDataMapSchema());
}
- @Override
- public void incrementalBuild(String[] segmentIds) {
- throw new UnsupportedOperationException();
- }
-
private DataMapFactory<? extends DataMap> createDataMapFactory()
throws MalformedDataMapCommandException {
CarbonTable mainTable = getMainTable();
@@ -135,12 +124,6 @@ public class IndexDataMapProvider extends DataMapProvider {
}
@Override
- public DataMapCatalog createDataMapCatalog() {
- // TODO create abstract class and move the default implementation there.
- return null;
- }
-
- @Override
public DataMapFactory getDataMapFactory() {
return dataMapFactory;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
index 8226f22..a1f80b1 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.DataMapCatalog;
import org.apache.carbondata.core.datamap.DataMapProvider;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProperty;
@@ -74,11 +73,6 @@ public class PreAggregateDataMapProvider extends DataMapProvider {
}
@Override
- public void initData() {
- // Nothing is needed to do by default
- }
-
- @Override
public void cleanMeta() {
DataMapSchema dataMapSchema = getDataMapSchema();
dropTableCommand = new CarbonDropTableCommand(
@@ -103,15 +97,6 @@ public class PreAggregateDataMapProvider extends DataMapProvider {
}
}
- @Override public void incrementalBuild(String[] segmentIds) {
- throw new UnsupportedOperationException();
- }
-
- @Override public DataMapCatalog createDataMapCatalog() {
- // TODO manage pre-agg also with catalog.
- return null;
- }
-
@Override
public DataMapFactory getDataMapFactory() {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index 9fdeb9d..d467017 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -109,8 +109,8 @@ case class CarbonAlterTableCompactionCommand(
compactionType = CompactionType.valueOf(alterTableModel.compactionType.toUpperCase)
} catch {
case _: Exception =>
- val alterTableCompactionExceptionEvent: AlterTableCompactionExceptionEvent =
- AlterTableCompactionExceptionEvent(sparkSession, table, alterTableModel)
+ val alterTableCompactionExceptionEvent: AlterTableCompactionAbortEvent =
+ AlterTableCompactionAbortEvent(sparkSession, table, alterTableModel)
OperationListenerBus.getInstance
.fireEvent(alterTableCompactionExceptionEvent, operationContext)
compactionException = operationContext.getProperty("compactionException").toString
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 6b1865a..86a2bc1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -236,12 +236,7 @@ case class CarbonLoadDataCommand(
val loadTablePreExecutionEvent: LoadTablePreExecutionEvent =
new LoadTablePreExecutionEvent(
table.getCarbonTableIdentifier,
- carbonLoadModel,
- factPath,
- dataFrame.isDefined,
- optionsFinal,
- options.asJava,
- isOverwriteTable)
+ carbonLoadModel)
operationContext.setProperty("isOverwrite", isOverwriteTable)
OperationListenerBus.getInstance.fireEvent(loadTablePreExecutionEvent, operationContext)
// Add pre event listener for index datamap
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
index e80579d..97eccc6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
@@ -122,51 +122,6 @@ object TimeSeriesUtil {
}
/**
- * Below method will be used to validate the hierarchy of time series and its value
- * validation will be done whether hierarchy order is proper or not and hierarchy level
- * value
- * TODO: we should remove this method
- *
- * @param timeSeriesHierarchyDetails
- * time series hierarchy string
- */
- @deprecated
- def validateAndGetTimeSeriesHierarchyDetails(timeSeriesHierarchyDetails: String): Array[
- (String, String)] = {
- val updatedtimeSeriesHierarchyDetails = timeSeriesHierarchyDetails.toLowerCase
- val timeSeriesHierarchy = updatedtimeSeriesHierarchyDetails.split(",")
- val hierBuffer = timeSeriesHierarchy.map {
- case f =>
- val splits = f.split("=")
- // checking hierarchy name is valid or not
- if (!TimeSeriesUDF.INSTANCE.TIMESERIES_FUNCTION.contains(splits(0).toLowerCase)) {
- throw new MalformedCarbonCommandException(s"Not supported hierarchy type: ${ splits(0) }")
- }
- // validating hierarchy level is valid or not
- if (!splits(1).equals("1")) {
- throw new MalformedCarbonCommandException(
- s"Unsupported Value for hierarchy:" +
- s"${ splits(0) }=${ splits(1) }")
- }
- (splits(0), splits(1))
- }
- // checking whether hierarchy is in proper order or not
- // get the index of first hierarchy
- val indexOfFirstHierarchy = TimeSeriesUDF.INSTANCE.TIMESERIES_FUNCTION
- .indexOf(hierBuffer(0)._1.toLowerCase)
- // now iterating through complete hierarchy to check any of the hierarchy index
- // is less than first one
- for (index <- 1 to hierBuffer.size - 1) {
- val currentIndex = TimeSeriesUDF.INSTANCE.TIMESERIES_FUNCTION
- .indexOf(hierBuffer(index)._1.toLowerCase)
- if (currentIndex < indexOfFirstHierarchy) {
- throw new MalformedCarbonCommandException(s"$timeSeriesHierarchyDetails is in wrong order")
- }
- }
- hierBuffer
- }
-
- /**
* Below method will be used to validate whether timeseries column present in
* select statement or not
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index 222d18d..8eb47fc 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -541,7 +541,6 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
// default value should not be converted to lower case
val tblProps = tblProp.get
.map(f => if (CarbonCommonConstants.TABLE_BLOCKSIZE.equalsIgnoreCase(f._1) ||
- CarbonCommonConstants.COLUMN_GROUPS.equalsIgnoreCase(f._1) ||
CarbonCommonConstants.SORT_COLUMNS.equalsIgnoreCase(f._1) ||
CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE.equalsIgnoreCase(f._1) ||
CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD.equalsIgnoreCase(f._1)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1413fd1..ec68c1d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -401,6 +401,8 @@
<exclude>**/*SparkUnknownExpression*.class</exclude>
<exclude>**/org/apache/carbondata/cluster/sdv/generated/*</exclude>
<exclude>**/org.apache.carbondata.cluster.sdv.generated.*</exclude>
+ <exclude>**/org.apache.spark.sql.test.*</exclude>
+ <exclude>**/org.apache.carbondata.format.*</exclude>
</excludes>
<includes>
<include>**/org.apache.*</include>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/pom.xml
----------------------------------------------------------------------
diff --git a/processing/pom.xml b/processing/pom.xml
index ab7c96c..d3cf77a 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -31,6 +31,7 @@
<properties>
<dev.path>${basedir}/../dev</dev.path>
+ <jacoco.append>true</jacoco.append>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/exception/DataLoadingException.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/exception/DataLoadingException.java b/processing/src/main/java/org/apache/carbondata/processing/exception/DataLoadingException.java
index 15ff95e..03c6c97 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/exception/DataLoadingException.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/exception/DataLoadingException.java
@@ -20,31 +20,7 @@ package org.apache.carbondata.processing.exception;
public class DataLoadingException extends Exception {
private static final long serialVersionUID = 1L;
- private long errorCode = -1;
-
- public DataLoadingException() {
- super();
- }
-
- public DataLoadingException(long errorCode, String message) {
- super(message);
- this.errorCode = errorCode;
- }
-
public DataLoadingException(String message) {
super(message);
}
-
- public DataLoadingException(Throwable cause) {
- super(cause);
- }
-
- public DataLoadingException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public long getErrorCode() {
- return errorCode;
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/exception/MultipleMatchingException.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/exception/MultipleMatchingException.java b/processing/src/main/java/org/apache/carbondata/processing/exception/MultipleMatchingException.java
index 6fa5e2c..b59732c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/exception/MultipleMatchingException.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/exception/MultipleMatchingException.java
@@ -20,31 +20,7 @@ package org.apache.carbondata.processing.exception;
public class MultipleMatchingException extends Exception {
private static final long serialVersionUID = 1L;
- private long errorCode = -1;
-
- public MultipleMatchingException() {
- super();
- }
-
- public MultipleMatchingException(long errorCode, String message) {
- super(message);
- this.errorCode = errorCode;
- }
-
public MultipleMatchingException(String message) {
super(message);
}
-
- public MultipleMatchingException(Throwable cause) {
- super(cause);
- }
-
- public MultipleMatchingException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public long getErrorCode() {
- return errorCode;
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/exception/SliceMergerException.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/exception/SliceMergerException.java b/processing/src/main/java/org/apache/carbondata/processing/exception/SliceMergerException.java
index d9640a9..a6f70c3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/exception/SliceMergerException.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/exception/SliceMergerException.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.processing.exception;
-import java.util.Locale;
-
public class SliceMergerException extends Exception {
/**
@@ -36,33 +34,12 @@ public class SliceMergerException extends Exception {
*
* @param msg The error message for this exception.
*/
- public SliceMergerException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
public SliceMergerException(String msg, Throwable t) {
super(msg, t);
this.msg = msg;
}
/**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
* getLocalizedMessage
*/
@Override public String getLocalizedMessage() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java b/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
index 2d96f6c..69f79f8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java
@@ -21,10 +21,8 @@ import java.io.IOException;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
@@ -63,7 +61,9 @@ public abstract class AbstractDataLoadProcessorStep {
* The output meta for this step. The data returns from this step is as per this meta.
*
*/
- public abstract DataField[] getOutput();
+ public DataField[] getOutput() {
+ return child.getOutput();
+ }
/**
* Initialization process for this step.
@@ -95,32 +95,7 @@ public abstract class AbstractDataLoadProcessorStep {
* @return Array of Iterator with data. It can be processed parallel if implementation class wants
* @throws CarbonDataLoadingException
*/
- public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
- Iterator<CarbonRowBatch>[] childIters = child.execute();
- Iterator<CarbonRowBatch>[] iterators = new Iterator[childIters.length];
- for (int i = 0; i < childIters.length; i++) {
- iterators[i] = getIterator(childIters[i]);
- }
- return iterators;
- }
-
- /**
- * Create the iterator using child iterator.
- *
- * @param childIter
- * @return new iterator with step specific processing.
- */
- protected Iterator<CarbonRowBatch> getIterator(final Iterator<CarbonRowBatch> childIter) {
- return new CarbonIterator<CarbonRowBatch>() {
- @Override public boolean hasNext() {
- return childIter.hasNext();
- }
-
- @Override public CarbonRowBatch next() {
- return processRowBatch(childIter.next());
- }
- };
- }
+ public abstract Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException;
/**
* Process the batch of rows as per the step logic.
@@ -131,20 +106,12 @@ public abstract class AbstractDataLoadProcessorStep {
protected CarbonRowBatch processRowBatch(CarbonRowBatch rowBatch) {
CarbonRowBatch newBatch = new CarbonRowBatch(rowBatch.getSize());
while (rowBatch.hasNext()) {
- newBatch.addRow(processRow(rowBatch.next()));
+ newBatch.addRow(null);
}
return newBatch;
}
/**
- * Process the row as per the step logic.
- *
- * @param row
- * @return processed row.
- */
- protected abstract CarbonRow processRow(CarbonRow row);
-
- /**
* Get the step name for logging purpose.
* @return Step name
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
index 50ebc34..1e53817 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.processing.loading.events;
-import java.util.Map;
-
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.events.Event;
@@ -32,24 +30,11 @@ public class LoadEvents {
public static class LoadTablePreExecutionEvent extends Event {
private CarbonTableIdentifier carbonTableIdentifier;
private CarbonLoadModel carbonLoadModel;
- private String factPath;
- private boolean isDataFrameDefined;
- private Map<String, String> optionsFinal;
- // userProvidedOptions are needed if we need only the load options given by user
- private Map<String, String> userProvidedOptions;
- private boolean isOverWriteTable;
public LoadTablePreExecutionEvent(CarbonTableIdentifier carbonTableIdentifier,
- CarbonLoadModel carbonLoadModel, String factPath, boolean isDataFrameDefined,
- Map<String, String> optionsFinal, Map<String, String> userProvidedOptions,
- boolean isOverWriteTable) {
+ CarbonLoadModel carbonLoadModel) {
this.carbonTableIdentifier = carbonTableIdentifier;
this.carbonLoadModel = carbonLoadModel;
- this.factPath = factPath;
- this.isDataFrameDefined = isDataFrameDefined;
- this.optionsFinal = optionsFinal;
- this.userProvidedOptions = userProvidedOptions;
- this.isOverWriteTable = isOverWriteTable;
}
public CarbonTableIdentifier getCarbonTableIdentifier() {
@@ -59,26 +44,6 @@ public class LoadEvents {
public CarbonLoadModel getCarbonLoadModel() {
return carbonLoadModel;
}
-
- public String getFactPath() {
- return factPath;
- }
-
- public boolean isDataFrameDefined() {
- return isDataFrameDefined;
- }
-
- public Map<String, String> getOptionsFinal() {
- return optionsFinal;
- }
-
- public Map<String, String> getUserProvidedOptions() {
- return userProvidedOptions;
- }
-
- public boolean isOverWriteTable() {
- return isOverWriteTable;
- }
}
/**
@@ -159,27 +124,4 @@ public class LoadEvents {
return carbonLoadModel;
}
}
-
- /**
- * Class for handling clean up in case of any failure and abort the operation.
- */
-
- public static class LoadTableAbortExecutionEvent extends Event {
- private CarbonTableIdentifier carbonTableIdentifier;
- private CarbonLoadModel carbonLoadModel;
- public LoadTableAbortExecutionEvent(CarbonTableIdentifier carbonTableIdentifier,
- CarbonLoadModel carbonLoadModel) {
- this.carbonTableIdentifier = carbonTableIdentifier;
- this.carbonLoadModel = carbonLoadModel;
- }
-
- public CarbonTableIdentifier getCarbonTableIdentifier() {
- return carbonTableIdentifier;
- }
-
- public CarbonLoadModel getCarbonLoadModel() {
- return carbonLoadModel;
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/exception/BadRecordFoundException.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/exception/BadRecordFoundException.java b/processing/src/main/java/org/apache/carbondata/processing/loading/exception/BadRecordFoundException.java
index 3c0fe53..b9a79ed 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/exception/BadRecordFoundException.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/exception/BadRecordFoundException.java
@@ -47,14 +47,6 @@ public class BadRecordFoundException extends CarbonDataLoadingException {
this.msg = msg;
}
- /**
- * Constructor
- *
- * @param t
- */
- public BadRecordFoundException(Throwable t) {
- super(t);
- }
/**
* getMessage
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/exception/NoRetryException.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/exception/NoRetryException.java b/processing/src/main/java/org/apache/carbondata/processing/loading/exception/NoRetryException.java
index 3533adb..41256a6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/exception/NoRetryException.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/exception/NoRetryException.java
@@ -40,25 +40,6 @@ public class NoRetryException extends RuntimeException {
}
/**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public NoRetryException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param t
- */
- public NoRetryException(Throwable t) {
- super(t);
- }
-
- /**
* getMessage
*/
public String getMessage() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java
index 23179fa..7b3c3df 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortScopeOptions.java
@@ -18,7 +18,6 @@
package org.apache.carbondata.processing.loading.sort;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonUtil;
/**
* Sort scope options
@@ -43,12 +42,8 @@ public class SortScopeOptions {
}
}
- public static boolean isValidSortOption(String sortScope) {
- return CarbonUtil.isValidSortOption(sortScope);
- }
-
public enum SortScope {
- NO_SORT, BATCH_SORT, LOCAL_SORT, GLOBAL_SORT;
+ NO_SORT, BATCH_SORT, LOCAL_SORT, GLOBAL_SORT
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
index fcc88b5..74e1594 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
@@ -157,14 +157,6 @@ public class ParallelReadMergeSorterImpl extends AbstractMergeSorter {
*/
private boolean processRowToNextStep(SortDataRows sortDataRows, SortParameters parameters)
throws CarbonDataLoadingException {
- if (null == sortDataRows) {
- LOGGER.info("Record Processed For table: " + parameters.getTableName());
- LOGGER.info("Number of Records was Zero");
- String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
- LOGGER.info(logMessage);
- return false;
- }
-
try {
// start sorting
sortDataRows.startSorting();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithColumnRangeImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithColumnRangeImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithColumnRangeImpl.java
index 808952b..5419e05 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithColumnRangeImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithColumnRangeImpl.java
@@ -168,14 +168,6 @@ public class ParallelReadMergeSorterWithColumnRangeImpl extends AbstractMergeSor
*/
private boolean processRowToNextStep(SortDataRows[] sortDataRows, SortParameters parameters)
throws CarbonDataLoadingException {
- if (null == sortDataRows || sortDataRows.length == 0) {
- LOGGER.info("Record Processed For table: " + parameters.getTableName());
- LOGGER.info("Number of Records was Zero");
- String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
- LOGGER.info(logMessage);
- return false;
- }
-
try {
for (int i = 0; i < sortDataRows.length; i++) {
// start sorting
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
index cb72f54..5cb099e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
@@ -309,14 +309,6 @@ public class UnsafeBatchParallelReadMergeSorterImpl extends AbstractMergeSorter
*/
private boolean processRowToNextStep(UnsafeSortDataRows sortDataRows, SortParameters parameters)
throws CarbonDataLoadingException {
- if (null == sortDataRows) {
- LOGGER.info("Record Processed For table: " + parameters.getTableName());
- LOGGER.info("Number of Records was Zero");
- String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
- LOGGER.info(logMessage);
- return false;
- }
-
try {
// start sorting
sortDataRows.startSorting();
@@ -333,6 +325,5 @@ public class UnsafeBatchParallelReadMergeSorterImpl extends AbstractMergeSorter
throw new CarbonDataLoadingException(e);
}
}
-
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterImpl.java
index e9cc986..afa30c0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterImpl.java
@@ -148,14 +148,6 @@ public class UnsafeParallelReadMergeSorterImpl extends AbstractMergeSorter {
*/
private boolean processRowToNextStep(UnsafeSortDataRows sortDataRows, SortParameters parameters)
throws CarbonDataLoadingException {
- if (null == sortDataRows) {
- LOGGER.info("Record Processed For table: " + parameters.getTableName());
- LOGGER.info("Number of Records was Zero");
- String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
- LOGGER.info(logMessage);
- return false;
- }
-
try {
// start sorting
sortDataRows.startSorting();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java
index da90630..5766105 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java
@@ -163,14 +163,6 @@ public class UnsafeParallelReadMergeSorterWithColumnRangeImpl extends AbstractMe
*/
private boolean processRowToNextStep(UnsafeSortDataRows[] sortDataRows, SortParameters parameters)
throws CarbonDataLoadingException {
- if (null == sortDataRows || sortDataRows.length == 0) {
- LOGGER.info("Record Processed For table: " + parameters.getTableName());
- LOGGER.info("Number of Records was Zero");
- String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
- LOGGER.info(logMessage);
- return false;
- }
-
try {
for (int i = 0; i < sortDataRows.length; i++) {
// start sorting
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
index e6e24ec..0b4eae2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
@@ -194,14 +194,6 @@ public class UnsafeIntermediateMerger {
mergerTask.add(executorService.submit(merger));
}
- private int getTotalNumberOfRows(List<UnsafeCarbonRowPage> unsafeCarbonRowPages) {
- int totalSize = 0;
- for (UnsafeCarbonRowPage unsafeCarbonRowPage : unsafeCarbonRowPages) {
- totalSize += unsafeCarbonRowPage.getBuffer().getActualSize();
- }
- return totalSize;
- }
-
public void finish() throws CarbonSortKeyAndGroupByException {
try {
executorService.shutdown();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
index 1224674..1a05b12 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -39,7 +39,6 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
-import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.exception.BadRecordFoundException;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
@@ -88,10 +87,6 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable());
}
- @Override public DataField[] getOutput() {
- return child.getOutput();
- }
-
@Override public void initialize() throws IOException {
super.initialize();
child.initialize();
@@ -173,8 +168,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
while (iterator.hasNext()) {
if (rowsNotExist) {
rowsNotExist = false;
- dataHandler = CarbonFactHandlerFactory
- .createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
+ dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model);
dataHandler.initialise();
}
processBatch(iterator.next(), dataHandler, iteratorIndex);
@@ -297,10 +291,6 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
rowCounter.getAndAdd(batch.getSize());
}
- @Override protected CarbonRow processRow(CarbonRow row) {
- return null;
- }
-
class DataWriterRunnable implements Runnable {
private Iterator<CarbonRowBatch> iterator;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java
index 43b2278..e3bc97f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.processing.loading.converter.BadRecordLogHolder;
import org.apache.carbondata.processing.loading.converter.FieldConverter;
import org.apache.carbondata.processing.loading.converter.RowConverter;
import org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl;
+import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.loading.partition.Partitioner;
import org.apache.carbondata.processing.loading.partition.impl.HashPartitionerImpl;
import org.apache.carbondata.processing.loading.partition.impl.RangePartitionerImpl;
@@ -64,11 +65,6 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
}
@Override
- public DataField[] getOutput() {
- return child.getOutput();
- }
-
- @Override
public void initialize() throws IOException {
super.initialize();
child.initialize();
@@ -156,14 +152,22 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
}
}
+ @Override public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
+ Iterator<CarbonRowBatch>[] childIters = child.execute();
+ Iterator<CarbonRowBatch>[] iterators = new Iterator[childIters.length];
+ for (int i = 0; i < childIters.length; i++) {
+ iterators[i] = getIterator(childIters[i]);
+ }
+ return iterators;
+ }
+
/**
* Create the iterator using child iterator.
*
* @param childIter
* @return new iterator with step specific processing.
*/
- @Override
- protected Iterator<CarbonRowBatch> getIterator(final Iterator<CarbonRowBatch> childIter) {
+ private Iterator<CarbonRowBatch> getIterator(final Iterator<CarbonRowBatch> childIter) {
return new CarbonIterator<CarbonRowBatch>() {
private boolean first = true;
private RowConverter localConverter;
@@ -209,11 +213,6 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
}
@Override
- protected CarbonRow processRow(CarbonRow row) {
- throw new UnsupportedOperationException();
- }
-
- @Override
public void close() {
if (!closed) {
if (null != badRecordLogger) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
index 9a11266..71a624e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
@@ -31,7 +31,6 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
-import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.exception.BadRecordFoundException;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
@@ -58,10 +57,6 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
CarbonUtil.getLocalDictionaryModel(configuration.getTableSpec().getCarbonTable());
}
- @Override public DataField[] getOutput() {
- return child.getOutput();
- }
-
@Override public void initialize() throws IOException {
super.initialize();
child.initialize();
@@ -94,10 +89,10 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
if (next.hasNext()) {
DataMapWriterListener listener = getDataMapWriterListener(0);
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
- .createCarbonFactDataHandlerModel(configuration, storeLocation, 0, k++, listener);
+ .createCarbonFactDataHandlerModel(configuration, storeLocation, i, k++, listener);
model.setColumnLocalDictGenMap(this.localDictionaryGeneratorMap);
CarbonFactHandler dataHandler = CarbonFactHandlerFactory
- .createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
+ .createCarbonFactHandler(model);
dataHandler.initialise();
processBatch(next, dataHandler);
finish(tableName, dataHandler);
@@ -157,9 +152,4 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
batch.close();
rowCounter.getAndAdd(batchSize);
}
-
- @Override protected CarbonRow processRow(CarbonRow row) {
- return null;
- }
-
}
[3/8] carbondata git commit: [CARBONDATA-2720] Remove dead code
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
deleted file mode 100644
index 77b5347..0000000
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.datamap.lucene;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datamap.dev.DataMapModel;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.solr.store.hdfs.HdfsDirectory;
-
-@InterfaceAudience.Internal
-public class LuceneCoarseGrainDataMap extends CoarseGrainDataMap {
-
- /**
- * log information
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(LuceneCoarseGrainDataMap.class.getName());
-
- public static final int BLOCKID_ID = 0;
-
- public static final int BLOCKLETID_ID = 1;
-
- public static final int PAGEID_ID = 2;
-
- public static final int ROWID_ID = 3;
- /**
- * searcher object for this datamap
- */
- private IndexSearcher indexSearcher = null;
-
- /**
- * default max values to return
- */
- private static int MAX_RESULT_NUMBER = 100;
-
- /**
- * analyzer for lucene index
- */
- private Analyzer analyzer;
-
- LuceneCoarseGrainDataMap(Analyzer analyzer) {
- this.analyzer = analyzer;
- }
-
- /**
- * It is called to load the data map to memory or to initialize it.
- */
- @Override
- public void init(DataMapModel dataMapModel) throws MemoryException, IOException {
- // get this path from file path
- Path indexPath = FileFactory.getPath(dataMapModel.getFilePath());
-
- LOGGER.info("Lucene index read path " + indexPath.toString());
-
- // get file system , use hdfs file system , realized in solr project
- FileSystem fs = FileFactory.getFileSystem(indexPath);
-
- // check this path valid
- if (!fs.exists(indexPath)) {
- String errorMessage = String.format("index directory %s not exists.", indexPath);
- LOGGER.error(errorMessage);
- throw new IOException(errorMessage);
- }
-
- if (!fs.isDirectory(indexPath)) {
- String errorMessage = String.format("error index path %s, must be directory", indexPath);
- LOGGER.error(errorMessage);
- throw new IOException(errorMessage);
- }
-
- // open this index path , use HDFS default configuration
- Directory indexDir = new HdfsDirectory(indexPath, FileFactory.getConfiguration());
-
- IndexReader indexReader = DirectoryReader.open(indexDir);
- if (indexReader == null) {
- throw new RuntimeException("failed to create index reader object");
- }
-
- // create a index searcher object
- indexSearcher = new IndexSearcher(indexReader);
- }
-
- /**
- * Prune the datamap with filter expression. It returns the list of
- * blocklets where these filters can exist.
- */
- @Override
- public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
- List<PartitionSpec> partitions) throws IOException {
-
- // convert filter expr into lucene list query
- List<String> fields = new ArrayList<String>();
-
- // only for test , query all data
- String strQuery = "*:*";
-
- String[] sFields = new String[fields.size()];
- fields.toArray(sFields);
-
- // get analyzer
- if (analyzer == null) {
- analyzer = new StandardAnalyzer();
- }
-
- // use MultiFieldQueryParser to parser query
- QueryParser queryParser = new MultiFieldQueryParser(sFields, analyzer);
- Query query;
- try {
- query = queryParser.parse(strQuery);
- } catch (ParseException e) {
- String errorMessage = String
- .format("failed to filter block with query %s, detail is %s", strQuery, e.getMessage());
- LOGGER.error(errorMessage);
- return null;
- }
-
- // execute index search
- TopDocs result;
- try {
- result = indexSearcher.search(query, MAX_RESULT_NUMBER);
- } catch (IOException e) {
- String errorMessage =
- String.format("failed to search lucene data, detail is %s", e.getMessage());
- LOGGER.error(errorMessage);
- throw new IOException(errorMessage);
- }
-
- // temporary data, delete duplicated data
- // Map<BlockId, Map<BlockletId, Map<PageId, Set<RowId>>>>
- Map<String, Set<Number>> mapBlocks = new HashMap<String, Set<Number>>();
-
- for (ScoreDoc scoreDoc : result.scoreDocs) {
- // get a document
- Document doc = indexSearcher.doc(scoreDoc.doc);
-
- // get all fields
- List<IndexableField> fieldsInDoc = doc.getFields();
-
- // get this block id Map<BlockId, Set<BlockletId>>>>
- String blockId = fieldsInDoc.get(BLOCKID_ID).stringValue();
- Set<Number> setBlocklets = mapBlocks.get(blockId);
- if (setBlocklets == null) {
- setBlocklets = new HashSet<Number>();
- mapBlocks.put(blockId, setBlocklets);
- }
-
- // get the blocklet id Set<BlockletId>
- Number blockletId = fieldsInDoc.get(BLOCKLETID_ID).numericValue();
- if (!setBlocklets.contains(blockletId.intValue())) {
- setBlocklets.add(blockletId.intValue());
- }
- }
-
- // result blocklets
- List<Blocklet> blocklets = new ArrayList<Blocklet>();
-
- // transform all blocks into result type blocklets Map<BlockId, Set<BlockletId>>
- for (Map.Entry<String, Set<Number>> mapBlock : mapBlocks.entrySet()) {
- String blockId = mapBlock.getKey();
- Set<Number> setBlocklets = mapBlock.getValue();
-
- // for blocklets in this block Set<BlockletId>
- for (Number blockletId : setBlocklets) {
-
- // add a CoarseGrainBlocklet
- blocklets.add(new Blocklet(blockId, blockletId.toString()));
- }
- }
-
- return blocklets;
- }
-
- @Override
- public boolean isScanRequired(FilterResolverIntf filterExp) {
- return true;
- }
-
- /**
- * Clear complete index table and release memory.
- */
- @Override
- public void clear() {
-
- }
-
- @Override
- public void finish() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
deleted file mode 100644
index b9c2ffa..0000000
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.datamap.lucene;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datamap.DataMapDistributable;
-import org.apache.carbondata.core.datamap.DataMapLevel;
-import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datamap.dev.DataMapModel;
-import org.apache.carbondata.core.datamap.dev.DataMapWriter;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
-import org.apache.carbondata.core.features.TableOperation;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-
-/**
- * CG level of lucene DataMap
- */
-@InterfaceAudience.Internal
-public class LuceneCoarseGrainDataMapFactory extends LuceneDataMapFactoryBase<CoarseGrainDataMap> {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(LuceneCoarseGrainDataMapFactory.class.getName());
-
- public LuceneCoarseGrainDataMapFactory(CarbonTable carbonTable, DataMapSchema dataMapSchema)
- throws MalformedDataMapCommandException {
- super(carbonTable, dataMapSchema);
- }
-
- /**
- * Get the datamap for segmentid
- */
- @Override
- public List<CoarseGrainDataMap> getDataMaps(Segment segment) throws IOException {
- List<CoarseGrainDataMap> lstDataMap = new ArrayList<>();
- CoarseGrainDataMap dataMap = new LuceneCoarseGrainDataMap(analyzer);
- try {
- dataMap.init(new DataMapModel(
- DataMapWriter.getDefaultDataMapPath(
- tableIdentifier.getTablePath(), segment.getSegmentNo(), dataMapName)));
- } catch (MemoryException e) {
- LOGGER.error("failed to get lucene datamap , detail is {}" + e.getMessage());
- return lstDataMap;
- }
- lstDataMap.add(dataMap);
- return lstDataMap;
- }
-
- /**
- * Get datamaps for distributable object.
- */
- @Override
- public List<CoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
- throws IOException {
- return getDataMaps(distributable.getSegment());
- }
-
- @Override
- public DataMapLevel getDataMapLevel() {
- return DataMapLevel.CG;
- }
-
- @Override
- public void deleteDatamapData() {
-
- }
-
- @Override
- public boolean willBecomeStale(TableOperation operation) {
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/datamap/mv/core/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/mv/core/pom.xml b/datamap/mv/core/pom.xml
index 54960a1..77b5cc7 100644
--- a/datamap/mv/core/pom.xml
+++ b/datamap/mv/core/pom.xml
@@ -31,6 +31,7 @@
<properties>
<dev.path>${basedir}/../../../dev</dev.path>
+ <jacoco.append>true</jacoco.append>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/MatchConditions.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/MatchConditions.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/MatchConditions.scala
deleted file mode 100644
index 2a4da27..0000000
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/rewrite/MatchConditions.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.mv.rewrite
-
-// TODO: implement this to modularize DefaultMatchingFunctions
-object MatchConditions {
-}
-
-class MatchConditions(flags: Long) {
- def hasFlag(flag: Long): Boolean = {
- throw new UnsupportedOperationException
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index 1d7fab3..592d831 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -31,6 +31,7 @@
<properties>
<dev.path>${basedir}/../dev</dev.path>
+ <jacoco.append>true</jacoco.append>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java
deleted file mode 100644
index 8be1e2e..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.hadoop;
-
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
-
-/**
- * CacheClient : Holds all the Cache access clients for Btree, Dictionary
- */
-public class CacheClient {
-
- // segment access client for driver LRU cache
- private CacheAccessClient<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper>
- segmentAccessClient;
-
- public CacheClient() {
- Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper> segmentCache =
- CacheProvider.getInstance().createCache(CacheType.DRIVER_BTREE);
- segmentAccessClient = new CacheAccessClient<>(segmentCache);
- }
-
- public CacheAccessClient<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper>
- getSegmentAccessClient() {
- return segmentAccessClient;
- }
-
- public void close() {
- segmentAccessClient.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java
deleted file mode 100644
index fe63e88..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.hadoop.internal;
-
-public enum CarbonFormatType {
- COLUMNAR
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java
deleted file mode 100644
index 23b39af..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.hadoop.internal;
-
-import org.apache.hadoop.mapreduce.InputSplit;
-
-/**
- * Carbon input split can be different format, application should create the record reader
- * based on format type.
- */
-public abstract class CarbonInputSplit extends InputSplit {
-
- /**
- * @return the format type of this split.
- */
- public abstract CarbonFormatType formatType();
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
deleted file mode 100644
index b619158..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.hadoop.internal.segment;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-
-/**
- * Within a carbon table, each data load becomes one Segment,
- * which stores all data files belong to this load in the segment folder.
- */
-public abstract class Segment {
-
- protected String id;
-
- /**
- * Path of the segment folder
- */
- private String path;
-
- public Segment(String id, String path) {
- this.id = id;
- this.path = path;
- }
-
- public String getId() {
- return id;
- }
-
- public String getPath() {
- return path;
- }
-
- /**
- * get all files, implementation may use the input filter and index to prune files
- * @param job job context
- * @param filterResolver filter
- * @return all files
- */
- public abstract List<InputSplit> getSplits(JobContext job, FilterResolverIntf filterResolver)
- throws IOException;
-
- /**
- * This is called after Segment is loaded and before segment is committed,
- * implementation should load indices if required
- * @param job job context
- */
- public abstract void setupForRead(JobContext job) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/hadoop/src/main/java/org/apache/carbondata/hadoop/testutil/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/testutil/StoreCreator.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/testutil/StoreCreator.java
index 63acad3..287f623 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/testutil/StoreCreator.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/testutil/StoreCreator.java
@@ -195,84 +195,70 @@ public class StoreCreator {
encodings.add(Encoding.DICTIONARY);
ColumnSchema id = new ColumnSchema();
id.setColumnName("ID");
- id.setColumnar(true);
id.setDataType(DataTypes.INT);
id.setEncodingList(encodings);
id.setColumnUniqueId(UUID.randomUUID().toString());
id.setColumnReferenceId(id.getColumnUniqueId());
id.setDimensionColumn(true);
- id.setColumnGroup(1);
columnSchemas.add(id);
ColumnSchema date = new ColumnSchema();
date.setColumnName("date");
- date.setColumnar(true);
date.setDataType(DataTypes.STRING);
date.setEncodingList(encodings);
date.setColumnUniqueId(UUID.randomUUID().toString());
date.setDimensionColumn(true);
- date.setColumnGroup(2);
date.setSortColumn(true);
date.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(date);
ColumnSchema country = new ColumnSchema();
country.setColumnName("country");
- country.setColumnar(true);
country.setDataType(DataTypes.STRING);
country.setEncodingList(encodings);
country.setColumnUniqueId(UUID.randomUUID().toString());
country.setDimensionColumn(true);
- country.setColumnGroup(3);
country.setSortColumn(true);
country.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(country);
ColumnSchema name = new ColumnSchema();
name.setColumnName("name");
- name.setColumnar(true);
name.setDataType(DataTypes.STRING);
name.setEncodingList(encodings);
name.setColumnUniqueId(UUID.randomUUID().toString());
name.setDimensionColumn(true);
- name.setColumnGroup(4);
name.setSortColumn(true);
name.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(name);
ColumnSchema phonetype = new ColumnSchema();
phonetype.setColumnName("phonetype");
- phonetype.setColumnar(true);
phonetype.setDataType(DataTypes.STRING);
phonetype.setEncodingList(encodings);
phonetype.setColumnUniqueId(UUID.randomUUID().toString());
phonetype.setDimensionColumn(true);
- phonetype.setColumnGroup(5);
phonetype.setSortColumn(true);
phonetype.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(phonetype);
ColumnSchema serialname = new ColumnSchema();
serialname.setColumnName("serialname");
- serialname.setColumnar(true);
serialname.setDataType(DataTypes.STRING);
serialname.setEncodingList(encodings);
serialname.setColumnUniqueId(UUID.randomUUID().toString());
serialname.setDimensionColumn(true);
- serialname.setColumnGroup(6);
serialname.setSortColumn(true);
serialname.setColumnReferenceId(id.getColumnUniqueId());
columnSchemas.add(serialname);
ColumnSchema salary = new ColumnSchema();
salary.setColumnName("salary");
- salary.setColumnar(true);
salary.setDataType(DataTypes.INT);
salary.setEncodingList(new ArrayList<Encoding>());
salary.setColumnUniqueId(UUID.randomUUID().toString());
salary.setDimensionColumn(false);
salary.setColumnReferenceId(id.getColumnUniqueId());
- salary.setColumnGroup(7);
columnSchemas.add(salary);
tableSchema.setListOfColumns(columnSchemas);
@@ -391,7 +377,6 @@ public class StoreCreator {
CarbonProperties.getInstance().addProperty("is.int.based.indexer", "true");
CarbonProperties.getInstance().addProperty("aggregate.columnar.keyblock", "true");
CarbonProperties.getInstance().addProperty("is.compressed.keyblock", "false");
- CarbonProperties.getInstance().addProperty("carbon.leaf.node.size", "120000");
String graphPath =
outPutLoc + File.separator + loadModel.getDatabaseName() + File.separator + tableName
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/hive/pom.xml
----------------------------------------------------------------------
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index c144353..081a5c0 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -32,6 +32,7 @@
<properties>
<hive.version>1.2.1</hive.version>
<dev.path>${basedir}/../../dev</dev.path>
+ <jacoco.append>true</jacoco.append>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index c61023a..5ede3e9 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -33,6 +33,7 @@
<properties>
<presto.version>0.187</presto.version>
<dev.path>${basedir}/../../dev</dev.path>
+ <jacoco.append>true</jacoco.append>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java
index 4ec145d..9be5560 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java
@@ -39,9 +39,7 @@ public class CarbondataColumnHandle implements ColumnHandle {
private final Type columnType;
private final int ordinalPosition;
private final int keyOrdinal;
- private final int columnGroupOrdinal;
- private final int columnGroupId;
private final String columnUniqueId;
private final boolean isInvertedIndex;
@@ -63,27 +61,13 @@ public class CarbondataColumnHandle implements ColumnHandle {
return keyOrdinal;
}
- public int getColumnGroupOrdinal() {
- return columnGroupOrdinal;
- }
-
- public int getColumnGroupId() {
- return columnGroupId;
- }
-
- public String getColumnUniqueId() {
- return columnUniqueId;
- }
-
@JsonCreator public CarbondataColumnHandle(
@JsonProperty("connectorId") String connectorId,
@JsonProperty("columnName") String columnName,
@JsonProperty("columnType") Type columnType,
@JsonProperty("ordinalPosition") int ordinalPosition,
@JsonProperty("keyOrdinal") int keyOrdinal,
- @JsonProperty("columnGroupOrdinal") int columnGroupOrdinal,
@JsonProperty("isMeasure") boolean isMeasure,
- @JsonProperty("columnGroupId") int columnGroupId,
@JsonProperty("columnUniqueId") String columnUniqueId,
@JsonProperty("isInvertedIndex") boolean isInvertedIndex,
@JsonProperty("precision") int precision,
@@ -94,10 +78,8 @@ public class CarbondataColumnHandle implements ColumnHandle {
this.ordinalPosition = requireNonNull(ordinalPosition, "ordinalPosition is null");
this.keyOrdinal = requireNonNull(keyOrdinal, "keyOrdinal is null");
- this.columnGroupOrdinal = requireNonNull(columnGroupOrdinal, "columnGroupOrdinal is null");
this.isMeasure = isMeasure;
- this.columnGroupId = requireNonNull(columnGroupId, "columnGroupId is null");
this.columnUniqueId = columnUniqueId;//requireNonNull(columnUniqueId, "columnUniqueId is null");
this.isInvertedIndex = requireNonNull(isInvertedIndex, "isInvertedIndex is null");
this.precision = precision;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
index 256e405..097d667 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
@@ -176,20 +176,18 @@ public class CarbondataMetadata implements ConnectorMetadata {
Type spiType = carbonDataType2SpiMapper(cs);
columnHandles.put(cs.getColumnName(),
- new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType,
- column.getSchemaOrdinal(), column.getKeyOrdinal(), column.getColumnGroupOrdinal(),
- false, cs.getColumnGroupId(), cs.getColumnUniqueId(), cs.isUseInvertedIndex(),
- cs.getPrecision(), cs.getScale()));
+ new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, column.getSchemaOrdinal(),
+ column.getKeyOrdinal(), false,
+ cs.getColumnUniqueId(), cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale()));
}
for (CarbonMeasure measure : cb.getMeasureByTableName(tableName)) {
ColumnSchema cs = measure.getColumnSchema();
Type spiType = carbonDataType2SpiMapper(cs);
columnHandles.put(cs.getColumnName(),
- new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType,
- cs.getSchemaOrdinal(), measure.getOrdinal(), cs.getColumnGroupId(), true,
- cs.getColumnGroupId(), cs.getColumnUniqueId(), cs.isUseInvertedIndex(),
- cs.getPrecision(), cs.getScale()));
+ new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, cs.getSchemaOrdinal(),
+ measure.getOrdinal(), true,
+ cs.getColumnUniqueId(), cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale()));
}
columnHandleMap = columnHandles.build();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index f4415b8..ffa5b00 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -155,13 +155,11 @@ object CarbonDataStoreCreator {
val id: ColumnSchema = new ColumnSchema()
id.setColumnName("ID")
- id.setColumnar(true)
id.setDataType(DataTypes.INT)
id.setEncodingList(dictionaryEncoding)
id.setColumnUniqueId(UUID.randomUUID().toString)
id.setColumnReferenceId(id.getColumnUniqueId)
id.setDimensionColumn(true)
- id.setColumnGroup(1)
id.setSchemaOrdinal(0)
columnSchemas.add(id)
@@ -172,94 +170,79 @@ object CarbonDataStoreCreator {
val date: ColumnSchema = new ColumnSchema()
date.setColumnName("date")
- date.setColumnar(true)
date.setDataType(DataTypes.DATE)
date.setEncodingList(directDictionaryEncoding)
date.setColumnUniqueId(UUID.randomUUID().toString)
date.setDimensionColumn(true)
- date.setColumnGroup(2)
date.setColumnReferenceId(date.getColumnUniqueId)
date.setSchemaOrdinal(1)
columnSchemas.add(date)
val country: ColumnSchema = new ColumnSchema()
country.setColumnName("country")
- country.setColumnar(true)
country.setDataType(DataTypes.STRING)
country.setEncodingList(dictionaryEncoding)
country.setColumnUniqueId(UUID.randomUUID().toString)
country.setColumnReferenceId(country.getColumnUniqueId)
country.setDimensionColumn(true)
- country.setColumnGroup(3)
country.setSchemaOrdinal(2)
country.setColumnReferenceId(country.getColumnUniqueId)
columnSchemas.add(country)
val name: ColumnSchema = new ColumnSchema()
name.setColumnName("name")
- name.setColumnar(true)
name.setDataType(DataTypes.STRING)
name.setEncodingList(dictionaryEncoding)
name.setColumnUniqueId(UUID.randomUUID().toString)
name.setDimensionColumn(true)
- name.setColumnGroup(4)
name.setSchemaOrdinal(3)
name.setColumnReferenceId(name.getColumnUniqueId)
columnSchemas.add(name)
val phonetype: ColumnSchema = new ColumnSchema()
phonetype.setColumnName("phonetype")
- phonetype.setColumnar(true)
phonetype.setDataType(DataTypes.STRING)
phonetype.setEncodingList(dictionaryEncoding)
phonetype.setColumnUniqueId(UUID.randomUUID().toString)
phonetype.setDimensionColumn(true)
- phonetype.setColumnGroup(5)
phonetype.setSchemaOrdinal(4)
phonetype.setColumnReferenceId(phonetype.getColumnUniqueId)
columnSchemas.add(phonetype)
val serialname: ColumnSchema = new ColumnSchema()
serialname.setColumnName("serialname")
- serialname.setColumnar(true)
serialname.setDataType(DataTypes.STRING)
serialname.setEncodingList(dictionaryEncoding)
serialname.setColumnUniqueId(UUID.randomUUID().toString)
serialname.setDimensionColumn(true)
- serialname.setColumnGroup(6)
serialname.setSchemaOrdinal(5)
serialname.setColumnReferenceId(serialname.getColumnUniqueId)
columnSchemas.add(serialname)
val salary: ColumnSchema = new ColumnSchema()
salary.setColumnName("salary")
- salary.setColumnar(true)
salary.setDataType(DataTypes.DOUBLE)
salary.setEncodingList(dictionaryEncoding)
salary.setColumnUniqueId(UUID.randomUUID().toString)
salary.setDimensionColumn(true)
- salary.setColumnGroup(7)
salary.setSchemaOrdinal(6)
salary.setColumnReferenceId(salary.getColumnUniqueId)
columnSchemas.add(salary)
val bonus: ColumnSchema = new ColumnSchema()
bonus.setColumnName("bonus")
- bonus.setColumnar(true)
bonus.setDataType(DataTypes.createDecimalType(10, 4))
bonus.setPrecision(10)
bonus.setScale(4)
bonus.setEncodingList(invertedIndexEncoding)
bonus.setColumnUniqueId(UUID.randomUUID().toString)
bonus.setDimensionColumn(false)
- bonus.setColumnGroup(8)
bonus.setSchemaOrdinal(7)
bonus.setColumnReferenceId(bonus.getColumnUniqueId)
columnSchemas.add(bonus)
val monthlyBonus: ColumnSchema = new ColumnSchema()
monthlyBonus.setColumnName("monthlyBonus")
- monthlyBonus.setColumnar(true)
monthlyBonus.setDataType(DataTypes.createDecimalType(18, 4))
monthlyBonus.setPrecision(18)
monthlyBonus.setScale(4)
@@ -267,42 +250,35 @@ object CarbonDataStoreCreator {
monthlyBonus.setEncodingList(invertedIndexEncoding)
monthlyBonus.setColumnUniqueId(UUID.randomUUID().toString)
monthlyBonus.setDimensionColumn(false)
- monthlyBonus.setColumnGroup(9)
monthlyBonus.setColumnReferenceId(monthlyBonus.getColumnUniqueId)
columnSchemas.add(monthlyBonus)
val dob: ColumnSchema = new ColumnSchema()
dob.setColumnName("dob")
- dob.setColumnar(true)
dob.setDataType(DataTypes.TIMESTAMP)
dob.setEncodingList(directDictionaryEncoding)
dob.setColumnUniqueId(UUID.randomUUID().toString)
dob.setDimensionColumn(true)
- dob.setColumnGroup(9)
dob.setSchemaOrdinal(9)
dob.setColumnReferenceId(dob.getColumnUniqueId)
columnSchemas.add(dob)
val shortField: ColumnSchema = new ColumnSchema()
shortField.setColumnName("shortField")
- shortField.setColumnar(true)
shortField.setDataType(DataTypes.SHORT)
shortField.setEncodingList(dictionaryEncoding)
shortField.setColumnUniqueId(UUID.randomUUID().toString)
shortField.setDimensionColumn(true)
- shortField.setColumnGroup(10)
shortField.setSchemaOrdinal(10)
shortField.setColumnReferenceId(shortField.getColumnUniqueId)
columnSchemas.add(shortField)
val isCurrentEmployee: ColumnSchema = new ColumnSchema()
isCurrentEmployee.setColumnName("isCurrentEmployee")
- isCurrentEmployee.setColumnar(true)
isCurrentEmployee.setDataType(DataTypes.BOOLEAN)
isCurrentEmployee.setEncodingList(invertedIndexEncoding)
isCurrentEmployee.setColumnUniqueId(UUID.randomUUID().toString)
isCurrentEmployee.setDimensionColumn(false)
- isCurrentEmployee.setColumnGroup(11)
isCurrentEmployee.setColumnReferenceId(isCurrentEmployee.getColumnUniqueId)
columnSchemas.add(isCurrentEmployee)
@@ -479,7 +455,6 @@ object CarbonDataStoreCreator {
CarbonProperties.getInstance
.addProperty("aggregate.columnar.keyblock", "true")
CarbonProperties.getInstance.addProperty("is.compressed.keyblock", "false")
- CarbonProperties.getInstance.addProperty("carbon.leaf.node.size", "120000")
CarbonProperties.getInstance
.addProperty("carbon.direct.dictionary", "true")
val graphPath: String = outPutLoc + File.separator + loadModel.getDatabaseName +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index a967806..ee87d92 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -31,6 +31,7 @@
<properties>
<dev.path>${basedir}/../../dev</dev.path>
+ <jacoco.append>true</jacoco.append>
<build.directory.projectCommon>../../common/target</build.directory.projectCommon>
<build.directory.projectCore>../../core/target</build.directory.projectCore>
<build.directory.projectProcessing>../../processing/target</build.directory.projectProcessing>
@@ -40,8 +41,8 @@
<build.directory.projectSpark2>../../integration/spark2/target</build.directory.projectSpark2>
<build.directory.projectSparkCommon>../../integration/spark-common/target</build.directory.projectSparkCommon>
<build.directory.projectSparkCommonTest>../../integration/spark-common-test/target</build.directory.projectSparkCommonTest>
- <build.directory.projectHive>../../integration/hive/target</build.directory.projectHive>
- <build.directory.projectPresto>../../integration/presto/target</build.directory.projectPresto>
+ <!--<build.directory.projectHive>../../integration/hive/target</build.directory.projectHive>-->
+ <!--<build.directory.projectPresto>../../integration/presto/target</build.directory.projectPresto>-->
<build.directory.projectStoreSdk>../../store/sdk/target</build.directory.projectStoreSdk>
<build.directory.projectStoreSearch>../../store/search/target</build.directory.projectStoreSearch>
<build.directory.projectStreaming>../../streaming/target</build.directory.projectStreaming>
@@ -57,8 +58,8 @@
<classes.directory.projectSpark2>../../integration/spark2/target/classes</classes.directory.projectSpark2>
<classes.directory.projectSparkCommon>../../integration/spark-common/target/classes</classes.directory.projectSparkCommon>
<classes.directory.projectSparkCommonTest>../../integration/spark-common-test/target/classes</classes.directory.projectSparkCommonTest>
- <classes.directory.projectHive>../../integration/hive/target/classes</classes.directory.projectHive>
- <classes.directory.projectPresto>../../integration/presto/target/classes</classes.directory.projectPresto>
+ <!--<classes.directory.projectHive>../../integration/hive/target/classes</classes.directory.projectHive>-->
+ <!--<classes.directory.projectPresto>../../integration/presto/target/classes</classes.directory.projectPresto>-->
<classes.directory.projectStoreSdk>../../store/sdk/target/classes</classes.directory.projectStoreSdk>
<classes.directory.projectStoreSearch>../../store/search/target/classes</classes.directory.projectStoreSearch>
<classes.directory.projectStreaming>../../streaming/target/classes</classes.directory.projectStreaming>
@@ -76,10 +77,10 @@
<sources.directory.projectSpark2>../../integration/spark2/src/main/scala</sources.directory.projectSpark2>
<sources.directory.projectSparkCommon>../../integration/spark-common/src/main/java</sources.directory.projectSparkCommon>
<sources.directory.projectSparkCommon>../../integration/spark-common/src/main/scala</sources.directory.projectSparkCommon>
- <sources.directory.projectHive>../../integration/hive/src/main/java</sources.directory.projectHive>
- <sources.directory.projectHive>../../integration/hive/src/main/scala</sources.directory.projectHive>
- <sources.directory.projectPresto>../../integration/presto/src/main/java</sources.directory.projectPresto>
- <sources.directory.projectPresto>../../integration/presto/src/main/scala</sources.directory.projectPresto>
+ <!--<sources.directory.projectHive>../../integration/hive/src/main/java</sources.directory.projectHive>-->
+ <!--<sources.directory.projectHive>../../integration/hive/src/main/scala</sources.directory.projectHive>-->
+ <!--<sources.directory.projectPresto>../../integration/presto/src/main/java</sources.directory.projectPresto>-->
+ <!--<sources.directory.projectPresto>../../integration/presto/src/main/scala</sources.directory.projectPresto>-->
<sources.directory.projectStoreSdk>../../store/sdk/src/main/java</sources.directory.projectStoreSdk>
<sources.directory.projectStoreSearch>../../store/search/src/main/java</sources.directory.projectStoreSearch>
<sources.directory.projectStoreSearch>../../store/search/src/main/scala</sources.directory.projectStoreSearch>
@@ -97,8 +98,8 @@
<generated-sources.directory.projectSpark2>../../integration/spark2/target/generated-sources/annotations</generated-sources.directory.projectSpark2>
<generated-sources.directory.projectSparkCommon>../../integration/spark-common/target/generated-sources/annotations</generated-sources.directory.projectSparkCommon>
<generated-sources.directory.projectSparkCommonTest>../../integration/spark-common-test/target/generated-sources/annotations</generated-sources.directory.projectSparkCommonTest>
- <generated-sources.directory.projectHive>../../integration/hive/target/generated-sources/annotations</generated-sources.directory.projectHive>
- <generated-sources.directory.projectPresto>../../integration/presto/target/generated-sources/annotations</generated-sources.directory.projectPresto>
+ <!--<generated-sources.directory.projectHive>../../integration/hive/target/generated-sources/annotations</generated-sources.directory.projectHive>-->
+ <!--<generated-sources.directory.projectPresto>../../integration/presto/target/generated-sources/annotations</generated-sources.directory.projectPresto>-->
<generated-sources.directory.projectStoreSdk>../../store/sdk/target/generated-sources/annotations</generated-sources.directory.projectStoreSdk>
<generated-sources.directory.projectStoreSearch>../../store/search/target/generated-sources/annotations</generated-sources.directory.projectStoreSearch>
<generated-sources.directory.projectStreaming>../../streaming/target/generated-sources/annotations</generated-sources.directory.projectStreaming>
@@ -324,12 +325,12 @@
<fileset dir="${build.directory.projectSparkCommonTest}">
<include name="jacoco.exec" />
</fileset>
- <fileset dir="${build.directory.projectHive}" erroronmissingdir="false">
+ <!--<fileset dir="${build.directory.projectHive}" erroronmissingdir="false">
<include name="jacoco.exec" />
- </fileset>
- <fileset dir="${build.directory.projectPresto}" erroronmissingdir="false">
+ </fileset>-->
+ <!--<fileset dir="${build.directory.projectPresto}" erroronmissingdir="false">
<include name="jacoco.exec" />
- </fileset>
+ </fileset>-->
<fileset dir="${build.directory.projectStoreSdk}" erroronmissingdir="false">
<include name="jacoco.exec" />
</fileset>
@@ -354,13 +355,13 @@
<fileset dir="${classes.directory.projectCore}" />
<fileset dir="${classes.directory.projectProcessing}" />
<fileset dir="${classes.directory.projectHadoop}" />
- <fileset dir="${classes.directory.projectFormat}" erroronmissingdir="false"/>
+ <!--<fileset dir="${classes.directory.projectFormat}" erroronmissingdir="false"/>-->
<fileset dir="${classes.directory.projectSpark}" erroronmissingdir="false"/>
<fileset dir="${classes.directory.projectSpark2}" erroronmissingdir="false"/>
<fileset dir="${classes.directory.projectSparkCommon}" />
<fileset dir="${classes.directory.projectSparkCommonTest}" />
- <fileset dir="${classes.directory.projectHive}" erroronmissingdir="false" />
- <fileset dir="${classes.directory.projectPresto}" erroronmissingdir="false" />
+ <!--<fileset dir="${classes.directory.projectHive}" erroronmissingdir="false" />-->
+ <!--<fileset dir="${classes.directory.projectPresto}" erroronmissingdir="false" />-->
<fileset dir="${classes.directory.projectStoreSdk}" erroronmissingdir="false" />
<fileset dir="${classes.directory.projectStoreSearch}" erroronmissingdir="false" />
<fileset dir="${classes.directory.projectStreaming}" erroronmissingdir="false" />
@@ -372,12 +373,12 @@
<fileset dir="${sources.directory.projectCore}" />
<fileset dir="${sources.directory.projectProcessing}" />
<fileset dir="${sources.directory.projectHadoop}" />
- <fileset dir="${sources.directory.projectFormat}" erroronmissingdir="false"/>
+ <!--<fileset dir="${sources.directory.projectFormat}" erroronmissingdir="false"/>-->
<fileset dir="${sources.directory.projectSpark}" erroronmissingdir="false"/>
<fileset dir="${sources.directory.projectSpark2}" erroronmissingdir="false"/>
<fileset dir="${sources.directory.projectSparkCommon}" />
- <fileset dir="${sources.directory.projectHive}" erroronmissingdir="false" />
- <fileset dir="${sources.directory.projectPresto}" erroronmissingdir="false" />
+ <!--<fileset dir="${sources.directory.projectHive}" erroronmissingdir="false" />-->
+ <!--<fileset dir="${sources.directory.projectPresto}" erroronmissingdir="false" />-->
<fileset dir="${sources.directory.projectStoreSdk}" erroronmissingdir="false" />
<fileset dir="${sources.directory.projectStoreSearch}" erroronmissingdir="false" />
<fileset dir="${sources.directory.projectStreaming}" erroronmissingdir="false" />
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common-test/src/test/resources/encoding_types.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/encoding_types.csv b/integration/spark-common-test/src/test/resources/encoding_types.csv
new file mode 100644
index 0000000..70a52ee
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/encoding_types.csv
@@ -0,0 +1,3 @@
+begin_time,name,begin_time1,begin_time2,begin_time3,begin_time4,begin_time5,begin_time6,begin_time7,begin_time8,begin_time9,begin_time10,begin_time11,begin_time12,begin_time13,begin_time14,begin_time15,begin_time16,begin_time17,begin_time18,begin_time19,begin_time20
+1497376581,name1,10000,8388600,125,1497376581,8386600,10000,100,125,1497376581,1497423738,2139095000,1497376581,1497423738,32000,123.4,11.1,3200.1,214744460.2,1497376581,1497376581
+1497408581,name2,32000,45000,25,10000,55000,32000,75,35,1497423838,1497423838,2147484000,1497423838,1497423838,31900,838860.7,12.3,127.1,214748360.2,1497408581,1497408581
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common-test/src/test/resources/short_int_as_target_type.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/short_int_as_target_type.csv b/integration/spark-common-test/src/test/resources/short_int_as_target_type.csv
deleted file mode 100644
index 964517e..0000000
--- a/integration/spark-common-test/src/test/resources/short_int_as_target_type.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-begin_time,name
-1497376581,name1
-1497423838,name2
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
index 3eaf0e8..13066eb 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
@@ -112,37 +112,49 @@ class IntegerDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
}
- test("short int as target type in deltaIntegerCodec") {
+ test("test all codecs") {
sql(
"""
- | DROP TABLE IF EXISTS short_int_target_table
+ | DROP TABLE IF EXISTS all_encoding_table
""".stripMargin)
//begin_time column will be encoded by deltaIntegerCodec
sql(
"""
- | CREATE TABLE short_int_target_table
- | (begin_time bigint, name string)
+ | CREATE TABLE all_encoding_table
+ | (begin_time bigint, name string,begin_time1 long,begin_time2 long,begin_time3 long,
+ | begin_time4 long,begin_time5 int,begin_time6 int,begin_time7 int,begin_time8 short,
+ | begin_time9 bigint,begin_time10 bigint,begin_time11 bigint,begin_time12 int,
+ | begin_time13 int,begin_time14 short,begin_time15 double,begin_time16 double,
+ | begin_time17 double,begin_time18 double,begin_time19 int,begin_time20 double)
| STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
+ """.stripMargin.replaceAll(System.lineSeparator, ""))
sql(
s"""
- | LOAD DATA LOCAL INPATH '$resourcesPath/short_int_as_target_type.csv'
- | INTO TABLE short_int_target_table
+ | LOAD DATA LOCAL INPATH '$resourcesPath/encoding_types.csv'
+ | INTO TABLE all_encoding_table
""".stripMargin)
checkAnswer(
- sql("select begin_time from short_int_target_table"),
- Seq(Row(1497376581), Row(1497423838))
+ sql("select begin_time from all_encoding_table"),
+ sql("select begin_time from all_encoding_table")
+ )
+
+ val ff = BigInt(2147484000L)
+ checkAnswer(
+ sql("select begin_time,begin_time1,begin_time2,begin_time3,begin_time4,begin_time5,begin_time6,begin_time7,begin_time8,begin_time9,begin_time10,begin_time11,begin_time12,begin_time13,begin_time14,begin_time15,begin_time16,begin_time17,begin_time18,begin_time19,begin_time20 from all_encoding_table"),
+ Seq(Row(1497376581,10000,8388600,125,1497376581,8386600,10000,100,125,1497376581,1497423738,2139095000,1497376581,1497423738,32000,123.4,11.1,3200.1,214744460.2,1497376581,1497376581),
+ Row(1497408581,32000,45000,25,10000,55000,32000,75,35,1497423838,1497423838,ff,1497423838,1497423838,31900,838860.7,12.3,127.1,214748360.2,1497408581,1497408581))
)
sql(
"""
- | DROP TABLE short_int_target_table
+ | DROP TABLE all_encoding_table
""".stripMargin)
}
+
test("Create a table that contains short data type") {
sql("CREATE TABLE if not exists short_table(col1 short, col2 BOOLEAN) STORED BY 'carbondata'")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala
index 28ff91a..2d5f05a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala
@@ -35,14 +35,22 @@ class DoubleDataTypeTest extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_sint")
sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_int")
sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_long")
+ sql("DROP TABLE IF EXISTS oscon_carbon_old1")
sql("""create table oscon_carbon_old (CUST_PRFRD_FLG String,PROD_BRAND_NAME String,PROD_COLOR String,CUST_LAST_RVW_DATE String,CUST_COUNTRY String,CUST_CITY String,PRODUCT_NAME String,CUST_JOB_TITLE String,CUST_STATE String,CUST_BUY_POTENTIAL String,PRODUCT_MODEL String,ITM_ID String,ITM_NAME String,PRMTION_ID String,PRMTION_NAME String,SHP_MODE_ID String,SHP_MODE String,DELIVERY_COUNTRY String,DELIVERY_STATE String,DELIVERY_CITY String,DELIVERY_DISTRICT String,ACTIVE_EMUI_VERSION String,WH_NAME String,STR_ORDER_DATE String,OL_ORDER_NO String,OL_ORDER_DATE String,OL_SITE String,CUST_FIRST_NAME String,CUST_LAST_NAME String,CUST_BIRTH_DY String,CUST_BIRTH_MM String,CUST_BIRTH_YR String,CUST_BIRTH_COUNTRY String,CUST_SEX String,CUST_ADDRESS_ID String,CUST_STREET_NO String,CUST_STREET_NAME String,CUST_AGE String,CUST_SUITE_NO String,CUST_ZIP String,CUST_COUNTY String,PRODUCT_ID String,PROD_SHELL_COLOR String,DEVICE_NAME String,PROD_SHORT_DESC String,PROD_LONG_DESC String,PROD_THUMB
String,PROD_IMAGE String,PROD_UPDATE_DATE String,PROD_LIVE String,PROD_LOC String,PROD_RAM String,PROD_ROM String,PROD_CPU_CLOCK String,PROD_SERIES String,ITM_REC_START_DATE String,ITM_REC_END_DATE String,ITM_BRAND_ID String,ITM_BRAND String,ITM_CLASS_ID String,ITM_CLASS String,ITM_CATEGORY_ID String,ITM_CATEGORY String,ITM_MANUFACT_ID String,ITM_MANUFACT String,ITM_FORMULATION String,ITM_COLOR String,ITM_CONTAINER String,ITM_MANAGER_ID String,PRM_START_DATE String,PRM_END_DATE String,PRM_CHANNEL_DMAIL String,PRM_CHANNEL_EMAIL String,PRM_CHANNEL_CAT String,PRM_CHANNEL_TV String,PRM_CHANNEL_RADIO String,PRM_CHANNEL_PRESS String,PRM_CHANNEL_EVENT String,PRM_CHANNEL_DEMO String,PRM_CHANNEL_DETAILS String,PRM_PURPOSE String,PRM_DSCNT_ACTIVE String,SHP_CODE String,SHP_CARRIER String,SHP_CONTRACT String,CHECK_DATE String,CHECK_YR String,CHECK_MM String,CHECK_DY String,CHECK_HOUR String,BOM String,INSIDE_NAME String,PACKING_DATE String,PACKING_YR String,PACKING_MM String,PACKING_DY String,
PACKING_HOUR String,DELIVERY_PROVINCE String,PACKING_LIST_NO String,ACTIVE_CHECK_TIME String,ACTIVE_CHECK_YR String,ACTIVE_CHECK_MM String,ACTIVE_CHECK_DY String,ACTIVE_CHECK_HOUR String,ACTIVE_AREA_ID String,ACTIVE_COUNTRY String,ACTIVE_PROVINCE String,ACTIVE_CITY String,ACTIVE_DISTRICT String,ACTIVE_NETWORK String,ACTIVE_FIRMWARE_VER String,ACTIVE_OS_VERSION String,LATEST_CHECK_TIME String,LATEST_CHECK_YR String,LATEST_CHECK_MM String,LATEST_CHECK_DY String,LATEST_CHECK_HOUR String,LATEST_AREAID String,LATEST_COUNTRY String,LATEST_PROVINCE String,LATEST_CITY String,LATEST_DISTRICT String,LATEST_FIRMWARE_VER String,LATEST_EMUI_VERSION String,LATEST_OS_VERSION String,LATEST_NETWORK String,WH_ID String,WH_STREET_NO String,WH_STREET_NAME String,WH_STREET_TYPE String,WH_SUITE_NO String,WH_CITY String,WH_COUNTY String,WH_STATE String,WH_ZIP String,WH_COUNTRY String,OL_SITE_DESC String,OL_RET_ORDER_NO String,OL_RET_DATE String,PROD_MODEL_ID String,CUST_ID String,PROD_UNQ_MDL_ID String,CU
ST_NICK_NAME String,CUST_LOGIN String,CUST_EMAIL_ADDR String,PROD_UNQ_DEVICE_ADDR String,PROD_UQ_UUID String,PROD_BAR_CODE String,TRACKING_NO String,STR_ORDER_NO String,CUST_DEP_COUNT double,CUST_VEHICLE_COUNT double,CUST_ADDRESS_CNT double,CUST_CRNT_CDEMO_CNT double,CUST_CRNT_HDEMO_CNT double,CUST_CRNT_ADDR_DM double,CUST_FIRST_SHIPTO_CNT double,CUST_FIRST_SALES_CNT double,CUST_GMT_OFFSET double,CUST_DEMO_CNT double,CUST_INCOME double,PROD_UNLIMITED double,PROD_OFF_PRICE double,PROD_UNITS double,TOTAL_PRD_COST double,TOTAL_PRD_DISC double,PROD_WEIGHT double,REG_UNIT_PRICE double,EXTENDED_AMT double,UNIT_PRICE_DSCNT_PCT double,DSCNT_AMT double,PROD_STD_CST double,TOTAL_TX_AMT double,FREIGHT_CHRG double,WAITING_PERIOD double,DELIVERY_PERIOD double,ITM_CRNT_PRICE double,ITM_UNITS double,ITM_WSLE_CST double,ITM_SIZE double,PRM_CST double,PRM_RESPONSE_TARGET double,PRM_ITM_DM double,SHP_MODE_CNT double,WH_GMT_OFFSET double,WH_SQ_FT double,STR_ORD_QTY double,STR_WSLE_CST double,STR_LIST_
PRICE double,STR_SALES_PRICE double,STR_EXT_DSCNT_AMT double,STR_EXT_SALES_PRICE double,STR_EXT_WSLE_CST double,STR_EXT_LIST_PRICE double,STR_EXT_TX double,STR_COUPON_AMT double,STR_NET_PAID double,STR_NET_PAID_INC_TX double,STR_NET_PRFT double,STR_SOLD_YR_CNT double,STR_SOLD_MM_CNT double,STR_SOLD_ITM_CNT double,STR_TOTAL_CUST_CNT double,STR_AREA_CNT double,STR_DEMO_CNT double,STR_OFFER_CNT double,STR_PRM_CNT double,STR_TICKET_CNT double,STR_NET_PRFT_DM_A double,STR_NET_PRFT_DM_B double,STR_NET_PRFT_DM_C double,STR_NET_PRFT_DM_D double,STR_NET_PRFT_DM_E double,STR_RET_STR_ID double,STR_RET_REASON_CNT double,STR_RET_TICKET_NO double,STR_RTRN_QTY double,STR_RTRN_AMT double,STR_RTRN_TX double,STR_RTRN_AMT_INC_TX double,STR_RET_FEE double,STR_RTRN_SHIP_CST double,STR_RFNDD_CSH double,STR_REVERSED_CHRG double,STR_STR_CREDIT double,STR_RET_NET_LOSS double,STR_RTRNED_YR_CNT double,STR_RTRN_MM_CNT double,STR_RET_ITM_CNT double,STR_RET_CUST_CNT double,STR_RET_AREA_CNT double,STR_RET_OFFER_C
NT double,STR_RET_PRM_CNT double,STR_RET_NET_LOSS_DM_A double,STR_RET_NET_LOSS_DM_B double,STR_RET_NET_LOSS_DM_C double,STR_RET_NET_LOSS_DM_D double,OL_ORD_QTY double,OL_WSLE_CST double,OL_LIST_PRICE double,OL_SALES_PRICE double,OL_EXT_DSCNT_AMT double,OL_EXT_SALES_PRICE double,OL_EXT_WSLE_CST double,OL_EXT_LIST_PRICE double,OL_EXT_TX double,OL_COUPON_AMT double,OL_EXT_SHIP_CST double,OL_NET_PAID double,OL_NET_PAID_INC_TX double,OL_NET_PAID_INC_SHIP double,OL_NET_PAID_INC_SHIP_TX double,OL_NET_PRFT double,OL_SOLD_YR_CNT double,OL_SOLD_MM_CNT double,OL_SHIP_DATE_CNT double,OL_ITM_CNT double,OL_BILL_CUST_CNT double,OL_BILL_AREA_CNT double,OL_BILL_DEMO_CNT double,OL_BILL_OFFER_CNT double,OL_SHIP_CUST_CNT double,OL_SHIP_AREA_CNT double,OL_SHIP_DEMO_CNT double,OL_SHIP_OFFER_CNT double,OL_WEB_PAGE_CNT double,OL_WEB_SITE_CNT double,OL_SHIP_MODE_CNT double,OL_WH_CNT double,OL_PRM_CNT double,OL_NET_PRFT_DM_A double,OL_NET_PRFT_DM_B double,OL_NET_PRFT_DM_C double,OL_NET_PRFT_DM_D double,OL_RE
T_RTRN_QTY double,OL_RTRN_AMT double,OL_RTRN_TX double,OL_RTRN_AMT_INC_TX double,OL_RET_FEE double,OL_RTRN_SHIP_CST double,OL_RFNDD_CSH double,OL_REVERSED_CHRG double,OL_ACCOUNT_CREDIT double,OL_RTRNED_YR_CNT double,OL_RTRNED_MM_CNT double,OL_RTRITM_CNT double,OL_RFNDD_CUST_CNT double,OL_RFNDD_AREA_CNT double,OL_RFNDD_DEMO_CNT double,OL_RFNDD_OFFER_CNT double,OL_RTRNING_CUST_CNT double,OL_RTRNING_AREA_CNT double,OL_RTRNING_DEMO_CNT double,OL_RTRNING_OFFER_CNT double,OL_RTRWEB_PAGE_CNT double,OL_REASON_CNT double,OL_NET_LOSS double,OL_NET_LOSS_DM_A double,OL_NET_LOSS_DM_B double,OL_NET_LOSS_DM_C double) STORED BY 'org.apache.carbondata.format' tblproperties('DICTIONARY_EXCLUDE'='CUST_ID,CUST_NICK_NAME,CUST_FIRST_NAME,CUST_LAST_NAME,CUST_LOGIN,CUST_EMAIL_ADDR,PROD_UNQ_MDL_ID,PROD_UNQ_DEVICE_ADDR,PROD_UQ_UUID,DEVICE_NAME,PROD_BAR_CODE,ITM_ID,ITM_NAME,ITM_BRAND_ID,ITM_BRAND,BOM,PACKING_LIST_NO,TRACKING_NO,ACTIVE_FIRMWARE_VER,LATEST_FIRMWARE_VER,LATEST_EMUI_VERSION,LATEST_NETWORK,STR_ORD
ER_NO','table_blocksize'='256')""")
sql(s"""load data LOCAL inpath '$resourcesPath/oscon_10.csv' into table oscon_carbon_old options('DELIMITER'=',', 'QUOTECHAR'='\"','FILEHEADER'='ACTIVE_AREA_ID, ACTIVE_CHECK_DY, ACTIVE_CHECK_HOUR, ACTIVE_CHECK_MM, ACTIVE_CHECK_TIME, ACTIVE_CHECK_YR, ACTIVE_CITY, ACTIVE_COUNTRY, ACTIVE_DISTRICT, ACTIVE_EMUI_VERSION, ACTIVE_FIRMWARE_VER, ACTIVE_NETWORK, ACTIVE_OS_VERSION, ACTIVE_PROVINCE, BOM, CHECK_DATE, CHECK_DY, CHECK_HOUR, CHECK_MM, CHECK_YR, CUST_ADDRESS_ID, CUST_AGE, CUST_BIRTH_COUNTRY, CUST_BIRTH_DY, CUST_BIRTH_MM, CUST_BIRTH_YR, CUST_BUY_POTENTIAL, CUST_CITY, CUST_STATE, CUST_COUNTRY, CUST_COUNTY, CUST_EMAIL_ADDR, CUST_LAST_RVW_DATE, CUST_FIRST_NAME, CUST_ID, CUST_JOB_TITLE, CUST_LAST_NAME, CUST_LOGIN, CUST_NICK_NAME, CUST_PRFRD_FLG, CUST_SEX, CUST_STREET_NAME, CUST_STREET_NO, CUST_SUITE_NO, CUST_ZIP, DELIVERY_CITY, DELIVERY_STATE, DELIVERY_COUNTRY, DELIVERY_DISTRICT, DELIVERY_PROVINCE, DEVICE_NAME, INSIDE_NAME, ITM_BRAND, ITM_BRAND_ID, ITM_CATEGORY, ITM_CATEGORY_ID, ITM_C
LASS, ITM_CLASS_ID, ITM_COLOR, ITM_CONTAINER, ITM_FORMULATION, ITM_MANAGER_ID, ITM_MANUFACT, ITM_MANUFACT_ID, ITM_ID, ITM_NAME, ITM_REC_END_DATE, ITM_REC_START_DATE, LATEST_AREAID, LATEST_CHECK_DY, LATEST_CHECK_HOUR, LATEST_CHECK_MM, LATEST_CHECK_TIME, LATEST_CHECK_YR, LATEST_CITY, LATEST_COUNTRY, LATEST_DISTRICT, LATEST_EMUI_VERSION, LATEST_FIRMWARE_VER, LATEST_NETWORK, LATEST_OS_VERSION, LATEST_PROVINCE, OL_ORDER_DATE, OL_ORDER_NO, OL_RET_ORDER_NO, OL_RET_DATE, OL_SITE, OL_SITE_DESC, PACKING_DATE, PACKING_DY, PACKING_HOUR, PACKING_LIST_NO, PACKING_MM, PACKING_YR, PRMTION_ID, PRMTION_NAME, PRM_CHANNEL_CAT, PRM_CHANNEL_DEMO, PRM_CHANNEL_DETAILS, PRM_CHANNEL_DMAIL, PRM_CHANNEL_EMAIL, PRM_CHANNEL_EVENT, PRM_CHANNEL_PRESS, PRM_CHANNEL_RADIO, PRM_CHANNEL_TV, PRM_DSCNT_ACTIVE, PRM_END_DATE, PRM_PURPOSE, PRM_START_DATE, PRODUCT_ID, PROD_BAR_CODE, PROD_BRAND_NAME, PRODUCT_NAME, PRODUCT_MODEL, PROD_MODEL_ID, PROD_COLOR, PROD_SHELL_COLOR, PROD_CPU_CLOCK, PROD_IMAGE, PROD_LIVE, PROD_LOC, PROD
_LONG_DESC, PROD_RAM, PROD_ROM, PROD_SERIES, PROD_SHORT_DESC, PROD_THUMB, PROD_UNQ_DEVICE_ADDR, PROD_UNQ_MDL_ID, PROD_UPDATE_DATE, PROD_UQ_UUID, SHP_CARRIER, SHP_CODE, SHP_CONTRACT, SHP_MODE_ID, SHP_MODE, STR_ORDER_DATE, STR_ORDER_NO, TRACKING_NO, WH_CITY, WH_COUNTRY, WH_COUNTY, WH_ID, WH_NAME, WH_STATE, WH_STREET_NAME, WH_STREET_NO, WH_STREET_TYPE, WH_SUITE_NO, WH_ZIP, CUST_DEP_COUNT, CUST_VEHICLE_COUNT, CUST_ADDRESS_CNT, CUST_CRNT_CDEMO_CNT, CUST_CRNT_HDEMO_CNT, CUST_CRNT_ADDR_DM, CUST_FIRST_SHIPTO_CNT, CUST_FIRST_SALES_CNT, CUST_GMT_OFFSET, CUST_DEMO_CNT, CUST_INCOME, PROD_UNLIMITED, PROD_OFF_PRICE, PROD_UNITS, TOTAL_PRD_COST, TOTAL_PRD_DISC, PROD_WEIGHT, REG_UNIT_PRICE, EXTENDED_AMT, UNIT_PRICE_DSCNT_PCT, DSCNT_AMT, PROD_STD_CST, TOTAL_TX_AMT, FREIGHT_CHRG, WAITING_PERIOD, DELIVERY_PERIOD, ITM_CRNT_PRICE, ITM_UNITS, ITM_WSLE_CST, ITM_SIZE, PRM_CST, PRM_RESPONSE_TARGET, PRM_ITM_DM, SHP_MODE_CNT, WH_GMT_OFFSET, WH_SQ_FT, STR_ORD_QTY, STR_WSLE_CST, STR_LIST_PRICE, STR_SALES_PRICE,
STR_EXT_DSCNT_AMT, STR_EXT_SALES_PRICE, STR_EXT_WSLE_CST, STR_EXT_LIST_PRICE, STR_EXT_TX, STR_COUPON_AMT, STR_NET_PAID, STR_NET_PAID_INC_TX, STR_NET_PRFT, STR_SOLD_YR_CNT, STR_SOLD_MM_CNT, STR_SOLD_ITM_CNT, STR_TOTAL_CUST_CNT, STR_AREA_CNT, STR_DEMO_CNT, STR_OFFER_CNT, STR_PRM_CNT, STR_TICKET_CNT, STR_NET_PRFT_DM_A, STR_NET_PRFT_DM_B, STR_NET_PRFT_DM_C, STR_NET_PRFT_DM_D, STR_NET_PRFT_DM_E, STR_RET_STR_ID, STR_RET_REASON_CNT, STR_RET_TICKET_NO, STR_RTRN_QTY, STR_RTRN_AMT, STR_RTRN_TX, STR_RTRN_AMT_INC_TX, STR_RET_FEE, STR_RTRN_SHIP_CST, STR_RFNDD_CSH, STR_REVERSED_CHRG, STR_STR_CREDIT, STR_RET_NET_LOSS, STR_RTRNED_YR_CNT, STR_RTRN_MM_CNT, STR_RET_ITM_CNT, STR_RET_CUST_CNT, STR_RET_AREA_CNT, STR_RET_OFFER_CNT, STR_RET_PRM_CNT, STR_RET_NET_LOSS_DM_A, STR_RET_NET_LOSS_DM_B, STR_RET_NET_LOSS_DM_C, STR_RET_NET_LOSS_DM_D, OL_ORD_QTY, OL_WSLE_CST, OL_LIST_PRICE, OL_SALES_PRICE, OL_EXT_DSCNT_AMT, OL_EXT_SALES_PRICE, OL_EXT_WSLE_CST, OL_EXT_LIST_PRICE, OL_EXT_TX, OL_COUPON_AMT, OL_EXT_SHIP_C
ST, OL_NET_PAID, OL_NET_PAID_INC_TX, OL_NET_PAID_INC_SHIP, OL_NET_PAID_INC_SHIP_TX, OL_NET_PRFT, OL_SOLD_YR_CNT, OL_SOLD_MM_CNT, OL_SHIP_DATE_CNT, OL_ITM_CNT, OL_BILL_CUST_CNT, OL_BILL_AREA_CNT, OL_BILL_DEMO_CNT, OL_BILL_OFFER_CNT, OL_SHIP_CUST_CNT, OL_SHIP_AREA_CNT, OL_SHIP_DEMO_CNT, OL_SHIP_OFFER_CNT, OL_WEB_PAGE_CNT, OL_WEB_SITE_CNT, OL_SHIP_MODE_CNT, OL_WH_CNT, OL_PRM_CNT, OL_NET_PRFT_DM_A, OL_NET_PRFT_DM_B, OL_NET_PRFT_DM_C, OL_NET_PRFT_DM_D, OL_RET_RTRN_QTY, OL_RTRN_AMT, OL_RTRN_TX, OL_RTRN_AMT_INC_TX, OL_RET_FEE, OL_RTRN_SHIP_CST, OL_RFNDD_CSH, OL_REVERSED_CHRG, OL_ACCOUNT_CREDIT, OL_RTRNED_YR_CNT, OL_RTRNED_MM_CNT, OL_RTRITM_CNT, OL_RFNDD_CUST_CNT, OL_RFNDD_AREA_CNT, OL_RFNDD_DEMO_CNT, OL_RFNDD_OFFER_CNT, OL_RTRNING_CUST_CNT, OL_RTRNING_AREA_CNT, OL_RTRNING_DEMO_CNT, OL_RTRNING_OFFER_CNT, OL_RTRWEB_PAGE_CNT, OL_REASON_CNT, OL_NET_LOSS, OL_NET_LOSS_DM_A, OL_NET_LOSS_DM_B, OL_NET_LOSS_DM_C','BAD_RECORDS_ACTION'='FORCE','BAD_RECORDS_LOGGER_ENABLE'='FALSE')""")
+ sql(
+ """create table oscon_carbon_old1 (CUST_PRFRD_FLG String,PROD_BRAND_NAME String,PROD_COLOR String,CUST_LAST_RVW_DATE String,CUST_COUNTRY String,CUST_CITY String,PRODUCT_NAME String,CUST_JOB_TITLE String,CUST_STATE String,CUST_BUY_POTENTIAL String,PRODUCT_MODEL String,ITM_ID String,ITM_NAME String,PRMTION_ID String,PRMTION_NAME String,SHP_MODE_ID String,SHP_MODE String,DELIVERY_COUNTRY String,DELIVERY_STATE String,DELIVERY_CITY String,DELIVERY_DISTRICT String,ACTIVE_EMUI_VERSION String,WH_NAME String,STR_ORDER_DATE String,OL_ORDER_NO String,OL_ORDER_DATE String,OL_SITE String,CUST_FIRST_NAME String,CUST_LAST_NAME String,CUST_BIRTH_DY String,CUST_BIRTH_MM String,CUST_BIRTH_YR String,CUST_BIRTH_COUNTRY String,CUST_SEX String,CUST_ADDRESS_ID String,CUST_STREET_NO String,CUST_STREET_NAME String,CUST_AGE String,CUST_SUITE_NO String,CUST_ZIP String,CUST_COUNTY String,PRODUCT_ID String,PROD_SHELL_COLOR String,DEVICE_NAME String,PROD_SHORT_DESC String,PROD_LONG_DESC String,PROD_THUMB S
tring,PROD_IMAGE String,PROD_UPDATE_DATE String,PROD_LIVE String,PROD_LOC String,PROD_RAM String,PROD_ROM String,PROD_CPU_CLOCK String,PROD_SERIES String,ITM_REC_START_DATE String,ITM_REC_END_DATE String,ITM_BRAND_ID String,ITM_BRAND String,ITM_CLASS_ID String,ITM_CLASS String,ITM_CATEGORY_ID String,ITM_CATEGORY String,ITM_MANUFACT_ID String,ITM_MANUFACT String,ITM_FORMULATION String,ITM_COLOR String,ITM_CONTAINER String,ITM_MANAGER_ID String,PRM_START_DATE String,PRM_END_DATE String,PRM_CHANNEL_DMAIL String,PRM_CHANNEL_EMAIL String,PRM_CHANNEL_CAT String,PRM_CHANNEL_TV String,PRM_CHANNEL_RADIO String,PRM_CHANNEL_PRESS String,PRM_CHANNEL_EVENT String,PRM_CHANNEL_DEMO String,PRM_CHANNEL_DETAILS String,PRM_PURPOSE String,PRM_DSCNT_ACTIVE String,SHP_CODE String,SHP_CARRIER String,SHP_CONTRACT String,CHECK_DATE String,CHECK_YR String,CHECK_MM String,CHECK_DY String,CHECK_HOUR String,BOM String,INSIDE_NAME String,PACKING_DATE String,PACKING_YR String,PACKING_MM String,PACKING_DY String,P
ACKING_HOUR String,DELIVERY_PROVINCE String,PACKING_LIST_NO String,ACTIVE_CHECK_TIME String,ACTIVE_CHECK_YR String,ACTIVE_CHECK_MM String,ACTIVE_CHECK_DY String,ACTIVE_CHECK_HOUR String,ACTIVE_AREA_ID String,ACTIVE_COUNTRY String,ACTIVE_PROVINCE String,ACTIVE_CITY String,ACTIVE_DISTRICT String,ACTIVE_NETWORK String,ACTIVE_FIRMWARE_VER String,ACTIVE_OS_VERSION String,LATEST_CHECK_TIME String,LATEST_CHECK_YR String,LATEST_CHECK_MM String,LATEST_CHECK_DY String,LATEST_CHECK_HOUR String,LATEST_AREAID String,LATEST_COUNTRY String,LATEST_PROVINCE String,LATEST_CITY String,LATEST_DISTRICT String,LATEST_FIRMWARE_VER String,LATEST_EMUI_VERSION String,LATEST_OS_VERSION String,LATEST_NETWORK String,WH_ID String,WH_STREET_NO String,WH_STREET_NAME String,WH_STREET_TYPE String,WH_SUITE_NO String,WH_CITY String,WH_COUNTY String,WH_STATE String,WH_ZIP String,WH_COUNTRY String,OL_SITE_DESC String,OL_RET_ORDER_NO String,OL_RET_DATE String,PROD_MODEL_ID String,CUST_ID String,PROD_UNQ_MDL_ID String,CUS
T_NICK_NAME String,CUST_LOGIN String,CUST_EMAIL_ADDR String,PROD_UNQ_DEVICE_ADDR String,PROD_UQ_UUID String,PROD_BAR_CODE String,TRACKING_NO String,STR_ORDER_NO String,CUST_DEP_COUNT double,CUST_VEHICLE_COUNT double,CUST_ADDRESS_CNT double,CUST_CRNT_CDEMO_CNT double,CUST_CRNT_HDEMO_CNT double,CUST_CRNT_ADDR_DM double,CUST_FIRST_SHIPTO_CNT double,CUST_FIRST_SALES_CNT double,CUST_GMT_OFFSET double,CUST_DEMO_CNT double,CUST_INCOME double,PROD_UNLIMITED double,PROD_OFF_PRICE double,PROD_UNITS double,TOTAL_PRD_COST double,TOTAL_PRD_DISC double,PROD_WEIGHT double,REG_UNIT_PRICE double,EXTENDED_AMT double,UNIT_PRICE_DSCNT_PCT double,DSCNT_AMT double,PROD_STD_CST double,TOTAL_TX_AMT double,FREIGHT_CHRG double,WAITING_PERIOD double,DELIVERY_PERIOD double,ITM_CRNT_PRICE double,ITM_UNITS double,ITM_WSLE_CST double,ITM_SIZE double,PRM_CST double,PRM_RESPONSE_TARGET double,PRM_ITM_DM double,SHP_MODE_CNT double,WH_GMT_OFFSET double,WH_SQ_FT double,STR_ORD_QTY double,STR_WSLE_CST double,STR_LIST_P
RICE double,STR_SALES_PRICE double,STR_EXT_DSCNT_AMT double,STR_EXT_SALES_PRICE double,STR_EXT_WSLE_CST double,STR_EXT_LIST_PRICE double,STR_EXT_TX double,STR_COUPON_AMT double,STR_NET_PAID double,STR_NET_PAID_INC_TX double,STR_NET_PRFT double,STR_SOLD_YR_CNT double,STR_SOLD_MM_CNT double,STR_SOLD_ITM_CNT double,STR_TOTAL_CUST_CNT double,STR_AREA_CNT double,STR_DEMO_CNT double,STR_OFFER_CNT double,STR_PRM_CNT double,STR_TICKET_CNT double,STR_NET_PRFT_DM_A double,STR_NET_PRFT_DM_B double,STR_NET_PRFT_DM_C double,STR_NET_PRFT_DM_D double,STR_NET_PRFT_DM_E double,STR_RET_STR_ID double,STR_RET_REASON_CNT double,STR_RET_TICKET_NO double,STR_RTRN_QTY double,STR_RTRN_AMT double,STR_RTRN_TX double,STR_RTRN_AMT_INC_TX double,STR_RET_FEE double,STR_RTRN_SHIP_CST double,STR_RFNDD_CSH double,STR_REVERSED_CHRG double,STR_STR_CREDIT double,STR_RET_NET_LOSS double,STR_RTRNED_YR_CNT double,STR_RTRN_MM_CNT double,STR_RET_ITM_CNT double,STR_RET_CUST_CNT double,STR_RET_AREA_CNT double,STR_RET_OFFER_CN
T double,STR_RET_PRM_CNT double,STR_RET_NET_LOSS_DM_A double,STR_RET_NET_LOSS_DM_B double,STR_RET_NET_LOSS_DM_C double,STR_RET_NET_LOSS_DM_D double,OL_ORD_QTY double,OL_WSLE_CST double,OL_LIST_PRICE double,OL_SALES_PRICE double,OL_EXT_DSCNT_AMT double,OL_EXT_SALES_PRICE double,OL_EXT_WSLE_CST double,OL_EXT_LIST_PRICE double,OL_EXT_TX double,OL_COUPON_AMT double,OL_EXT_SHIP_CST double,OL_NET_PAID double,OL_NET_PAID_INC_TX double,OL_NET_PAID_INC_SHIP double,OL_NET_PAID_INC_SHIP_TX double,OL_NET_PRFT double,OL_SOLD_YR_CNT double,OL_SOLD_MM_CNT double,OL_SHIP_DATE_CNT double,OL_ITM_CNT double,OL_BILL_CUST_CNT double,OL_BILL_AREA_CNT double,OL_BILL_DEMO_CNT double,OL_BILL_OFFER_CNT double,OL_SHIP_CUST_CNT double,OL_SHIP_AREA_CNT double,OL_SHIP_DEMO_CNT double,OL_SHIP_OFFER_CNT double,OL_WEB_PAGE_CNT double,OL_WEB_SITE_CNT double,OL_SHIP_MODE_CNT double,OL_WH_CNT double,OL_PRM_CNT double,OL_NET_PRFT_DM_A double,OL_NET_PRFT_DM_B double,OL_NET_PRFT_DM_C double,OL_NET_PRFT_DM_D double,OL_RET
_RTRN_QTY double,OL_RTRN_AMT double,OL_RTRN_TX double,OL_RTRN_AMT_INC_TX double,OL_RET_FEE double,OL_RTRN_SHIP_CST double,OL_RFNDD_CSH double,OL_REVERSED_CHRG double,OL_ACCOUNT_CREDIT double,OL_RTRNED_YR_CNT double,OL_RTRNED_MM_CNT double,OL_RTRITM_CNT double,OL_RFNDD_CUST_CNT double,OL_RFNDD_AREA_CNT double,OL_RFNDD_DEMO_CNT double,OL_RFNDD_OFFER_CNT double,OL_RTRNING_CUST_CNT double,OL_RTRNING_AREA_CNT double,OL_RTRNING_DEMO_CNT double,OL_RTRNING_OFFER_CNT double,OL_RTRWEB_PAGE_CNT double,OL_REASON_CNT double,OL_NET_LOSS double,OL_NET_LOSS_DM_A double,OL_NET_LOSS_DM_B double,OL_NET_LOSS_DM_C double) STORED BY 'org.apache.carbondata.format' tblproperties('DICTIONARY_INCLUDE'='CUST_DEP_COUNT','DICTIONARY_EXCLUDE'='CUST_ID,CUST_NICK_NAME,CUST_FIRST_NAME,CUST_LAST_NAME,CUST_LOGIN,CUST_EMAIL_ADDR,PROD_UNQ_MDL_ID,PROD_UNQ_DEVICE_ADDR,PROD_UQ_UUID,DEVICE_NAME,PROD_BAR_CODE,ITM_ID,ITM_NAME,ITM_BRAND_ID,ITM_BRAND,BOM,PACKING_LIST_NO,TRACKING_NO,ACTIVE_FIRMWARE_VER,LATEST_FIRMWARE_VER,LATES
T_EMUI_VERSION,LATEST_NETWORK,STR_ORDER_NO','table_blocksize'='256')""".stripMargin)
+ sql(s"""load data LOCAL inpath '$resourcesPath/oscon_10.csv' into table oscon_carbon_old1 options('DELIMITER'=',', 'QUOTECHAR'='\"','FILEHEADER'='ACTIVE_AREA_ID, ACTIVE_CHECK_DY, ACTIVE_CHECK_HOUR, ACTIVE_CHECK_MM, ACTIVE_CHECK_TIME, ACTIVE_CHECK_YR, ACTIVE_CITY, ACTIVE_COUNTRY, ACTIVE_DISTRICT, ACTIVE_EMUI_VERSION, ACTIVE_FIRMWARE_VER, ACTIVE_NETWORK, ACTIVE_OS_VERSION, ACTIVE_PROVINCE, BOM, CHECK_DATE, CHECK_DY, CHECK_HOUR, CHECK_MM, CHECK_YR, CUST_ADDRESS_ID, CUST_AGE, CUST_BIRTH_COUNTRY, CUST_BIRTH_DY, CUST_BIRTH_MM, CUST_BIRTH_YR, CUST_BUY_POTENTIAL, CUST_CITY, CUST_STATE, CUST_COUNTRY, CUST_COUNTY, CUST_EMAIL_ADDR, CUST_LAST_RVW_DATE, CUST_FIRST_NAME, CUST_ID, CUST_JOB_TITLE, CUST_LAST_NAME, CUST_LOGIN, CUST_NICK_NAME, CUST_PRFRD_FLG, CUST_SEX, CUST_STREET_NAME, CUST_STREET_NO, CUST_SUITE_NO, CUST_ZIP, DELIVERY_CITY, DELIVERY_STATE, DELIVERY_COUNTRY, DELIVERY_DISTRICT, DELIVERY_PROVINCE, DEVICE_NAME, INSIDE_NAME, ITM_BRAND, ITM_BRAND_ID, ITM_CATEGORY, ITM_CATEGORY_ID, ITM_
CLASS, ITM_CLASS_ID, ITM_COLOR, ITM_CONTAINER, ITM_FORMULATION, ITM_MANAGER_ID, ITM_MANUFACT, ITM_MANUFACT_ID, ITM_ID, ITM_NAME, ITM_REC_END_DATE, ITM_REC_START_DATE, LATEST_AREAID, LATEST_CHECK_DY, LATEST_CHECK_HOUR, LATEST_CHECK_MM, LATEST_CHECK_TIME, LATEST_CHECK_YR, LATEST_CITY, LATEST_COUNTRY, LATEST_DISTRICT, LATEST_EMUI_VERSION, LATEST_FIRMWARE_VER, LATEST_NETWORK, LATEST_OS_VERSION, LATEST_PROVINCE, OL_ORDER_DATE, OL_ORDER_NO, OL_RET_ORDER_NO, OL_RET_DATE, OL_SITE, OL_SITE_DESC, PACKING_DATE, PACKING_DY, PACKING_HOUR, PACKING_LIST_NO, PACKING_MM, PACKING_YR, PRMTION_ID, PRMTION_NAME, PRM_CHANNEL_CAT, PRM_CHANNEL_DEMO, PRM_CHANNEL_DETAILS, PRM_CHANNEL_DMAIL, PRM_CHANNEL_EMAIL, PRM_CHANNEL_EVENT, PRM_CHANNEL_PRESS, PRM_CHANNEL_RADIO, PRM_CHANNEL_TV, PRM_DSCNT_ACTIVE, PRM_END_DATE, PRM_PURPOSE, PRM_START_DATE, PRODUCT_ID, PROD_BAR_CODE, PROD_BRAND_NAME, PRODUCT_NAME, PRODUCT_MODEL, PROD_MODEL_ID, PROD_COLOR, PROD_SHELL_COLOR, PROD_CPU_CLOCK, PROD_IMAGE, PROD_LIVE, PROD_LOC, PRO
D_LONG_DESC, PROD_RAM, PROD_ROM, PROD_SERIES, PROD_SHORT_DESC, PROD_THUMB, PROD_UNQ_DEVICE_ADDR, PROD_UNQ_MDL_ID, PROD_UPDATE_DATE, PROD_UQ_UUID, SHP_CARRIER, SHP_CODE, SHP_CONTRACT, SHP_MODE_ID, SHP_MODE, STR_ORDER_DATE, STR_ORDER_NO, TRACKING_NO, WH_CITY, WH_COUNTRY, WH_COUNTY, WH_ID, WH_NAME, WH_STATE, WH_STREET_NAME, WH_STREET_NO, WH_STREET_TYPE, WH_SUITE_NO, WH_ZIP, CUST_DEP_COUNT, CUST_VEHICLE_COUNT, CUST_ADDRESS_CNT, CUST_CRNT_CDEMO_CNT, CUST_CRNT_HDEMO_CNT, CUST_CRNT_ADDR_DM, CUST_FIRST_SHIPTO_CNT, CUST_FIRST_SALES_CNT, CUST_GMT_OFFSET, CUST_DEMO_CNT, CUST_INCOME, PROD_UNLIMITED, PROD_OFF_PRICE, PROD_UNITS, TOTAL_PRD_COST, TOTAL_PRD_DISC, PROD_WEIGHT, REG_UNIT_PRICE, EXTENDED_AMT, UNIT_PRICE_DSCNT_PCT, DSCNT_AMT, PROD_STD_CST, TOTAL_TX_AMT, FREIGHT_CHRG, WAITING_PERIOD, DELIVERY_PERIOD, ITM_CRNT_PRICE, ITM_UNITS, ITM_WSLE_CST, ITM_SIZE, PRM_CST, PRM_RESPONSE_TARGET, PRM_ITM_DM, SHP_MODE_CNT, WH_GMT_OFFSET, WH_SQ_FT, STR_ORD_QTY, STR_WSLE_CST, STR_LIST_PRICE, STR_SALES_PRICE,
STR_EXT_DSCNT_AMT, STR_EXT_SALES_PRICE, STR_EXT_WSLE_CST, STR_EXT_LIST_PRICE, STR_EXT_TX, STR_COUPON_AMT, STR_NET_PAID, STR_NET_PAID_INC_TX, STR_NET_PRFT, STR_SOLD_YR_CNT, STR_SOLD_MM_CNT, STR_SOLD_ITM_CNT, STR_TOTAL_CUST_CNT, STR_AREA_CNT, STR_DEMO_CNT, STR_OFFER_CNT, STR_PRM_CNT, STR_TICKET_CNT, STR_NET_PRFT_DM_A, STR_NET_PRFT_DM_B, STR_NET_PRFT_DM_C, STR_NET_PRFT_DM_D, STR_NET_PRFT_DM_E, STR_RET_STR_ID, STR_RET_REASON_CNT, STR_RET_TICKET_NO, STR_RTRN_QTY, STR_RTRN_AMT, STR_RTRN_TX, STR_RTRN_AMT_INC_TX, STR_RET_FEE, STR_RTRN_SHIP_CST, STR_RFNDD_CSH, STR_REVERSED_CHRG, STR_STR_CREDIT, STR_RET_NET_LOSS, STR_RTRNED_YR_CNT, STR_RTRN_MM_CNT, STR_RET_ITM_CNT, STR_RET_CUST_CNT, STR_RET_AREA_CNT, STR_RET_OFFER_CNT, STR_RET_PRM_CNT, STR_RET_NET_LOSS_DM_A, STR_RET_NET_LOSS_DM_B, STR_RET_NET_LOSS_DM_C, STR_RET_NET_LOSS_DM_D, OL_ORD_QTY, OL_WSLE_CST, OL_LIST_PRICE, OL_SALES_PRICE, OL_EXT_DSCNT_AMT, OL_EXT_SALES_PRICE, OL_EXT_WSLE_CST, OL_EXT_LIST_PRICE, OL_EXT_TX, OL_COUPON_AMT, OL_EXT_SHIP_
CST, OL_NET_PAID, OL_NET_PAID_INC_TX, OL_NET_PAID_INC_SHIP, OL_NET_PAID_INC_SHIP_TX, OL_NET_PRFT, OL_SOLD_YR_CNT, OL_SOLD_MM_CNT, OL_SHIP_DATE_CNT, OL_ITM_CNT, OL_BILL_CUST_CNT, OL_BILL_AREA_CNT, OL_BILL_DEMO_CNT, OL_BILL_OFFER_CNT, OL_SHIP_CUST_CNT, OL_SHIP_AREA_CNT, OL_SHIP_DEMO_CNT, OL_SHIP_OFFER_CNT, OL_WEB_PAGE_CNT, OL_WEB_SITE_CNT, OL_SHIP_MODE_CNT, OL_WH_CNT, OL_PRM_CNT, OL_NET_PRFT_DM_A, OL_NET_PRFT_DM_B, OL_NET_PRFT_DM_C, OL_NET_PRFT_DM_D, OL_RET_RTRN_QTY, OL_RTRN_AMT, OL_RTRN_TX, OL_RTRN_AMT_INC_TX, OL_RET_FEE, OL_RTRN_SHIP_CST, OL_RFNDD_CSH, OL_REVERSED_CHRG, OL_ACCOUNT_CREDIT, OL_RTRNED_YR_CNT, OL_RTRNED_MM_CNT, OL_RTRITM_CNT, OL_RFNDD_CUST_CNT, OL_RFNDD_AREA_CNT, OL_RFNDD_DEMO_CNT, OL_RFNDD_OFFER_CNT, OL_RTRNING_CUST_CNT, OL_RTRNING_AREA_CNT, OL_RTRNING_DEMO_CNT, OL_RTRNING_OFFER_CNT, OL_RTRWEB_PAGE_CNT, OL_REASON_CNT, OL_NET_LOSS, OL_NET_LOSS_DM_A, OL_NET_LOSS_DM_B, OL_NET_LOSS_DM_C','BAD_RECORDS_ACTION'='FORCE','BAD_RECORDS_LOGGER_ENABLE'='FALSE')""")
}
test("test to check result for double data type") {
val result = sql("select OL_SALES_PRICE from oscon_carbon_old limit 10").count()
assert(result.equals(10L))
}
+ test("test to check result for double data type as dimension") {
+ val result = sql("select CUST_DEP_COUNT from oscon_carbon_old1 limit 10").count()
+ assert(result.equals(10L))
+ }
test("Double Datatype Check with AdaptiveDeltaFloating Codec (BYTE)") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
index 937aee9..0dba467 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
@@ -49,6 +49,9 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA local inpath '$resourcesPath/decimalBoundaryDataHive.csv' INTO table hiveBigDecimal")
sql("create table if not exists carbonBigDecimal_2 (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(30, 10)) STORED BY 'org.apache.carbondata.format'")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/decimalBoundaryDataCarbon.csv' into table carbonBigDecimal_2")
+
+ sql("create table if not exists carbonBigDecimal_3 (ID Int, date Timestamp, country String,name String, phonetype String, serialname String, salary decimal(30, 2)) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='salary')")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/decimalBoundaryDataCarbon.csv' into table carbonBigDecimal_3")
}
test("test detail query on big decimal column") {
@@ -188,6 +191,11 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
sql("select avg(salary)/10 from hiveBigDecimal"))
}
+ test("test lower precision definiton on big decimal column with high precision value") {
+ checkAnswer(sql("select count(salary) from carbonBigDecimal_3"),
+ sql("select count(salary) from hiveBigDecimal"))
+ }
+
test("test decimal compression where both precision and data falls in integer range") {
sql("create table decimal_int_test(d1 decimal(9,3)) stored by 'carbondata'")
sql(s"load data inpath '$resourcesPath/decimal_int_range.csv' into table decimal_int_test")
@@ -204,6 +212,7 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists hiveBigDecimal")
sql("drop table if exists carbonBigDecimal_2")
sql("DROP TABLE IF EXISTS decimal_int_test")
+ sql("drop table if exists carbonBigDecimal_3")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.SORT_SIZE,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index 5ab156c..b483594 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -21,13 +21,9 @@ import scala.collection.JavaConverters._
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper
-import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.hadoop.CacheClient
import org.apache.spark.sql.test.util.QueryTest
/**
@@ -138,11 +134,6 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
assert(segments.contains("2.1"))
assert(!segments.contains("2"))
assert(!segments.contains("3"))
- val cacheClient = new CacheClient()
- val segmentIdentifier = new TableSegmentUniqueIdentifier(absoluteTableIdentifier, "2")
- val wrapper: SegmentTaskIndexWrapper = cacheClient.getSegmentAccessClient.
- getIfPresent(segmentIdentifier)
- assert(null == wrapper)
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index f4c12f7..9866683 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -491,6 +491,8 @@ class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
sql("select * from normal_test where name='n502670' and city='c2670'"))
+ checkAnswer(sql("select * from datamap_test where name='n502670' or city='c2670'"),
+ sql("select * from normal_test where name='n502670' or city='c2670'"))
}
test("test invisible datamap during query") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/integration/spark-common/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index 599c6c8..0bac9f7 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -31,6 +31,7 @@
<properties>
<dev.path>${basedir}/../../dev</dev.path>
+ <jacoco.append>true</jacoco.append>
</properties>
<dependencies>
[8/8] carbondata git commit: [CARBONDATA-2720] Remove dead code
Posted by gv...@apache.org.
[CARBONDATA-2720] Remove dead code
For acturate coverage results and easy maintainance
This closes #2354
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f9114036
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f9114036
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f9114036
Branch: refs/heads/master
Commit: f911403673fdae89c8537293ed55ef515b1362ef
Parents: d267c40
Author: sraghunandan <ca...@gmail.com>
Authored: Thu Jul 12 12:23:24 2018 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Jul 12 12:23:24 2018 +0530
----------------------------------------------------------------------
.../sql/MalformedCarbonCommandException.java | 8 -
.../common/logging/LogServiceFactory.java | 5 -
core/pom.xml | 1 +
.../carbondata/core/cache/CacheProvider.java | 4 -
.../apache/carbondata/core/cache/CacheType.java | 16 +-
.../AbstractColumnDictionaryInfo.java | 18 -
.../cache/dictionary/ColumnDictionaryInfo.java | 13 -
.../dictionary/ColumnReverseDictionaryInfo.java | 18 +
.../dictionary/DictionaryBuilderException.java | 18 -
.../dictionary/ManageDictionaryAndBTree.java | 48 --
.../BlockletLevelDeleteDeltaDataCache.java | 56 ---
.../core/constants/CarbonCommonConstants.java | 2 -
.../core/datamap/DataMapProvider.java | 10 +-
.../datamap/DistributableDataMapFormat.java | 26 -
.../dev/expr/DataMapDistributableWrapper.java | 4 -
.../core/datastore/SegmentTaskIndexStore.java | 481 -------------------
.../datastore/TableSegmentUniqueIdentifier.java | 53 --
.../carbondata/core/datastore/TableSpec.java | 42 +-
.../core/datastore/block/AbstractIndex.java | 5 -
.../core/datastore/block/SegmentProperties.java | 224 +--------
.../core/datastore/block/SegmentTaskIndex.java | 53 --
.../block/SegmentTaskIndexWrapper.java | 129 -----
.../datastore/chunk/DimensionColumnPage.java | 16 +-
.../impl/FixedLengthDimensionColumnPage.java | 16 +-
.../impl/VariableLengthDimensionColumnPage.java | 16 +-
.../chunk/store/ColumnPageWrapper.java | 12 +-
.../datastore/columnar/ColumnGroupModel.java | 88 ----
.../datastore/columnar/ColumnWithRowId.java | 14 -
.../core/datastore/page/EncodedTablePage.java | 12 -
.../page/encoding/DefaultEncodingFactory.java | 7 +-
.../adaptive/AdaptiveFloatingCodec.java | 6 -
.../dimension/legacy/IndexStorageEncoder.java | 4 -
.../core/datastore/page/key/TablePageKey.java | 16 -
.../core/datastore/row/CarbonRow.java | 8 -
.../devapi/DictionaryGenerationException.java | 52 --
.../core/devapi/GeneratingBiDictionary.java | 48 --
.../dictionary/client/DictionaryClient.java | 6 +-
.../generator/TableDictionaryGenerator.java | 4 -
.../generator/key/DictionaryMessage.java | 5 -
.../dictionary/server/DictionaryServer.java | 18 +-
.../service/DictionaryServiceProvider.java | 2 +-
.../InvalidConfigurationException.java | 29 --
.../core/indexstore/schema/CarbonRowSchema.java | 4 +-
.../core/keygenerator/KeyGenerator.java | 7 -
.../DirectDictionaryKeyGeneratorFactory.java | 1 -
.../mdkey/AbstractKeyGenerator.java | 30 --
.../keygenerator/mdkey/NumberCompressor.java | 34 --
.../carbondata/core/locks/CarbonLockUtil.java | 4 +-
.../carbondata/core/locks/ZookeeperInit.java | 4 -
.../core/memory/HeapMemoryAllocator.java | 1 -
.../carbondata/core/memory/MemoryBlock.java | 2 +-
.../carbondata/core/memory/MemoryLocation.java | 9 -
.../compressor/ChunkCompressorMeta.java | 34 --
.../ThriftWrapperSchemaConverterImpl.java | 6 +-
.../datatype/DecimalConverterFactory.java | 14 -
.../core/metadata/schema/table/CarbonTable.java | 37 +-
.../schema/table/TableSchemaBuilder.java | 1 -
.../schema/table/column/CarbonColumn.java | 13 -
.../schema/table/column/CarbonDimension.java | 21 +-
.../table/column/CarbonImplicitDimension.java | 9 +-
.../schema/table/column/ColumnSchema.java | 33 --
.../core/mutate/CarbonUpdateUtil.java | 11 +-
.../mutate/data/DeleteDeltaCacheLoaderIntf.java | 27 --
.../core/readcommitter/ReadCommittedScope.java | 15 +-
.../scan/collector/ResultCollectorFactory.java | 3 +
.../exception/QueryExecutionException.java | 46 --
.../executor/impl/AbstractQueryExecutor.java | 19 +-
.../scan/executor/infos/BlockExecutionInfo.java | 20 -
.../scan/executor/infos/KeyStructureInfo.java | 98 ----
.../core/scan/executor/util/QueryUtil.java | 176 +------
.../exception/FilterIllegalMemberException.java | 29 --
.../exception/FilterUnsupportedException.java | 20 -
.../expression/logical/FalseExpression.java | 5 +-
.../expression/logical/RangeExpression.java | 5 +-
.../scan/expression/logical/TrueExpression.java | 5 +-
.../carbondata/core/scan/filter/FilterUtil.java | 4 +-
.../executer/RangeValueFilterExecuterImpl.java | 2 +-
.../executer/RowLevelFilterExecuterImpl.java | 42 +-
.../executer/ValueBasedFilterExecuterImpl.java | 49 --
.../scan/filter/intf/FilterOptimizerImpl.java | 33 --
.../filter/optimizer/RangeFilterOptmizer.java | 5 +-
.../TrueConditionalResolverImpl.java | 1 -
.../core/scan/result/BlockletScannedResult.java | 24 +-
.../result/impl/FilterQueryScannedResult.java | 6 +-
.../impl/NonFilterQueryScannedResult.java | 3 +-
.../scan/scanner/impl/BlockletFullScanner.java | 1 -
.../core/scan/wrappers/ByteArrayWrapper.java | 7 -
.../DriverQueryStatisticsRecorderImpl.java | 18 +-
.../core/stats/QueryStatisticsModel.java | 1 -
.../SegmentUpdateStatusManager.java | 4 +-
.../util/AbstractDataFileFooterConverter.java | 2 -
.../carbondata/core/util/CarbonMergerUtil.java | 65 ---
.../core/util/CarbonMetadataUtil.java | 28 +-
.../carbondata/core/util/CarbonProperties.java | 4 -
.../apache/carbondata/core/util/CarbonUtil.java | 69 +--
.../core/util/DataFileFooterConverter2.java | 7 +-
.../apache/carbondata/core/util/NodeHolder.java | 457 ------------------
.../carbondata/core/util/TaskMetricsMap.java | 4 -
.../core/util/path/CarbonTablePath.java | 50 --
.../carbondata/core/writer/ByteArrayHolder.java | 75 ---
.../core/writer/CarbonDeleteDeltaWriter.java | 7 -
.../writer/CarbonDeleteDeltaWriterImpl.java | 32 --
.../carbondata/core/writer/ThriftWriter.java | 20 -
.../carbondata/events/OperationListenerBus.java | 34 --
.../core/cache/CacheProviderTest.java | 34 --
.../dictionary/AbstractDictionaryCacheTest.java | 6 +-
.../datastore/SegmentTaskIndexStoreTest.java | 90 ----
.../datastore/block/SegmentPropertiesTest.java | 31 +-
.../block/SegmentPropertiesTestUtil.java | 15 -
.../datastore/block/SegmentTaskIndexTest.java | 77 ---
.../impl/FixedLengthDimensionDataChunkTest.java | 10 +-
.../datastore/page/encoding/RLECodecSuite.java | 158 ------
.../datastore/page/encoding/RLECodecTest.java | 158 ++++++
.../dictionary/client/DictionaryClientTest.java | 4 +-
...ncrementalColumnDictionaryGeneratorTest.java | 4 +-
.../ServerDictionaryGeneratorTest.java | 4 +-
.../generator/TableDictionaryGeneratorTest.java | 4 +-
.../core/metadata/CarbonMetadataTest.java | 13 +-
.../ThriftWrapperSchemaConverterImplTest.java | 130 -----
.../metadata/schema/table/CarbonTableTest.java | 3 +-
.../table/CarbonTableWithComplexTypesTest.java | 5 -
.../impl/RawBasedResultCollectorTest.java | 130 -----
.../core/scan/executor/util/QueryUtilTest.java | 36 --
.../scan/executor/util/RestructureUtilTest.java | 10 +-
.../scan/expression/ColumnExpressionTest.java | 1 +
.../conditional/EqualToExpressionUnitTest.java | 51 ++
.../GreaterThanEqualToExpressionUnitTest.java | 205 ++++++++
.../GreaterThanExpressionUnitTest.java | 203 ++++++++
.../LessThanEqualToExpressionUnitTest.java | 175 +++++++
.../conditional/LessThanExpressionUnitTest.java | 204 ++++++++
.../conditional/ListExpressionUnitTest.java | 22 +
.../NotEqualsExpressionUnitTest.java | 204 +++++++-
.../conditional/NotInExpressionUnitTest.java | 30 ++
.../expression/logical/AndExpressionTest.java | 68 +++
.../expression/logical/FalseExpressionTest.java | 12 +
.../expression/logical/OrExpressionTest.java | 68 +++
.../expression/logical/RangeExpressionTest.java | 148 ++++++
.../expression/logical/TrueExpressionTest.java | 57 +++
.../filter/FilterExpressionProcessorTest.java | 1 -
.../core/scan/filter/FilterUtilTest.java | 8 +-
.../executer/IncludeFilterExecuterImplTest.java | 2 +-
.../scan/wrappers/ByteArrayWrapperTest.java | 57 +++
.../core/util/CarbonMergerUtilTest.java | 41 --
.../carbondata/core/util/CarbonUtilTest.java | 53 +-
.../core/util/RangeFilterProcessorTest.java | 8 +-
.../writer/CarbonDictionaryWriterImplTest.java | 6 +-
.../lucene/LuceneCoarseGrainDataMap.java | 238 ---------
.../lucene/LuceneCoarseGrainDataMapFactory.java | 95 ----
datamap/mv/core/pom.xml | 1 +
.../carbondata/mv/rewrite/MatchConditions.scala | 28 --
hadoop/pom.xml | 1 +
.../apache/carbondata/hadoop/CacheClient.java | 48 --
.../hadoop/internal/CarbonFormatType.java | 22 -
.../hadoop/internal/CarbonInputSplit.java | 32 --
.../hadoop/internal/segment/Segment.java | 70 ---
.../hadoop/testutil/StoreCreator.java | 15 -
integration/hive/pom.xml | 1 +
integration/presto/pom.xml | 1 +
.../presto/CarbondataColumnHandle.java | 18 -
.../carbondata/presto/CarbondataMetadata.java | 14 +-
.../presto/util/CarbonDataStoreCreator.scala | 25 -
integration/spark-common-test/pom.xml | 41 +-
.../src/test/resources/encoding_types.csv | 3 +
.../test/resources/short_int_as_target_type.csv | 3 -
.../aggquery/IntegerDataTypeTestCase.scala | 32 +-
.../allqueries/DoubleDataTypeTest.scala | 8 +
.../testsuite/bigdecimal/TestBigDecimal.scala | 9 +
.../MajorCompactionIgnoreInMinorTest.scala | 9 -
.../testsuite/datamap/FGDataMapTestCase.scala | 2 +
integration/spark-common/pom.xml | 1 +
.../carbondata/events/AlterTableEvents.scala | 17 +-
.../carbondata/events/CleanFilesEvents.scala | 8 -
.../apache/carbondata/spark/CarbonOption.scala | 5 -
.../load/DataLoadProcessorStepOnSpark.scala | 3 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 3 -
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 17 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala | 17 -
.../carbondata/spark/util/CarbonScalaUtil.scala | 88 ----
.../carbondata/spark/util/CommonUtil.scala | 155 ------
.../spark/util/DataTypeConverterUtil.scala | 22 -
.../spark/util/GlobalDictionaryUtil.scala | 72 ---
.../streaming/StreamSinkFactory.scala | 9 +-
.../apache/spark/rdd/UpdateCoalescedRDD.scala | 89 ----
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 76 ---
.../command/carbonTableSchemaCommon.scala | 30 --
.../streaming/CarbonAppendableStreamSink.scala | 7 +-
integration/spark2/pom.xml | 1 +
.../datamap/IndexDataMapProvider.java | 17 -
.../datamap/PreAggregateDataMapProvider.java | 15 -
.../CarbonAlterTableCompactionCommand.scala | 4 +-
.../management/CarbonLoadDataCommand.scala | 7 +-
.../command/timeseries/TimeSeriesUtil.scala | 45 --
.../sql/parser/CarbonSpark2SqlParser.scala | 1 -
pom.xml | 2 +
processing/pom.xml | 1 +
.../exception/DataLoadingException.java | 24 -
.../exception/MultipleMatchingException.java | 24 -
.../exception/SliceMergerException.java | 23 -
.../loading/AbstractDataLoadProcessorStep.java | 43 +-
.../processing/loading/events/LoadEvents.java | 60 +--
.../exception/BadRecordFoundException.java | 8 -
.../loading/exception/NoRetryException.java | 19 -
.../loading/sort/SortScopeOptions.java | 7 +-
.../sort/impl/ParallelReadMergeSorterImpl.java | 8 -
...allelReadMergeSorterWithColumnRangeImpl.java | 8 -
.../UnsafeBatchParallelReadMergeSorterImpl.java | 9 -
.../impl/UnsafeParallelReadMergeSorterImpl.java | 8 -
...allelReadMergeSorterWithColumnRangeImpl.java | 8 -
.../unsafe/merger/UnsafeIntermediateMerger.java | 8 -
.../CarbonRowDataWriterProcessorStepImpl.java | 12 +-
.../steps/DataConverterProcessorStepImpl.java | 23 +-
.../steps/DataWriterBatchProcessorStepImpl.java | 14 +-
.../steps/DataWriterProcessorStepImpl.java | 11 +-
.../loading/steps/InputProcessorStepImpl.java | 4 -
.../InputProcessorStepWithNoConverterImpl.java | 4 -
.../steps/JsonInputProcessorStepImpl.java | 5 -
.../loading/steps/SortProcessorStepImpl.java | 12 -
.../processing/merger/CarbonCompactionUtil.java | 32 --
.../processing/merger/CarbonDataMergerUtil.java | 17 +-
.../merger/CompactionResultSortProcessor.java | 3 +-
.../partition/impl/DefaultLoadBalancer.java | 63 ---
.../partition/impl/PartitionMultiFileImpl.java | 44 --
.../impl/SampleDataPartitionerImpl.java | 40 --
.../partition/spliter/RowResultProcessor.java | 7 +-
.../exception/AlterPartitionSliceException.java | 78 ---
.../processing/splits/TableSplit.java | 124 -----
.../store/CarbonDataFileAttributes.java | 4 -
.../store/CarbonFactDataHandlerColumnar.java | 24 +-
.../store/CarbonFactDataHandlerModel.java | 26 -
.../store/CarbonFactHandlerFactory.java | 15 +-
.../util/CarbonDataProcessorUtil.java | 45 +-
231 files changed, 2025 insertions(+), 6223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
index 5fe3ce8..0394d29 100644
--- a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
+++ b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
@@ -59,14 +59,6 @@ public class MalformedCarbonCommandException extends Exception {
}
/**
- * getLocalizedMessage
- */
- @Override
- public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
* getMessage
*/
public String getMessage() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/common/src/main/java/org/apache/carbondata/common/logging/LogServiceFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/logging/LogServiceFactory.java b/common/src/main/java/org/apache/carbondata/common/logging/LogServiceFactory.java
index 410844e..cf84d2d 100644
--- a/common/src/main/java/org/apache/carbondata/common/logging/LogServiceFactory.java
+++ b/common/src/main/java/org/apache/carbondata/common/logging/LogServiceFactory.java
@@ -36,9 +36,4 @@ public final class LogServiceFactory {
public static LogService getLogService(final String className) {
return new StandardLogService(className);
}
-
- public static LogService getLogService() {
- return new StandardLogService();
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index c145c3b..71fa66c 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -31,6 +31,7 @@
<properties>
<dev.path>${basedir}/../dev</dev.path>
+ <jacoco.append>true</jacoco.append>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
index 0ee4f25..292d9eb 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
@@ -28,7 +28,6 @@ import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentif
import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache;
import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
import org.apache.carbondata.core.indexstore.BlockletDataMapIndexStore;
import org.apache.carbondata.core.util.CarbonProperties;
@@ -140,9 +139,6 @@ public class CacheProvider {
} else if (cacheType.equals(CacheType.FORWARD_DICTIONARY)) {
cacheObject =
new ForwardDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonLRUCache);
- } else if (cacheType.equals(cacheType.DRIVER_BTREE)) {
- cacheObject =
- new SegmentTaskIndexStore(carbonLRUCache);
} else if (cacheType.equals(cacheType.DRIVER_BLOCKLET_DATAMAP)) {
cacheObject = new BlockletDataMapIndexStore(carbonLRUCache);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java b/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
index 9cc2320..df32b6e 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
@@ -21,8 +21,6 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
-import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
/**
* class which defines different cache types. cache type can be dictionary cache for
@@ -46,19 +44,7 @@ public class CacheType<K, V> {
/**
* Executor BTree cache which maintains size of BTree metadata
*/
- public static final CacheType<TableBlockUniqueIdentifier, AbstractIndex> EXECUTOR_BTREE =
- new CacheType("executor_btree");
-
- /**
- * Executor BTree cache which maintains size of BTree metadata
- */
- public static final CacheType<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper>
- DRIVER_BTREE = new CacheType("driver_btree");
-
- /**
- * Executor BTree cache which maintains size of BTree metadata
- */
- public static final CacheType<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper>
+ public static final CacheType<TableSegmentUniqueIdentifier, AbstractIndex>
DRIVER_BLOCKLET_DATAMAP = new CacheType("driver_blocklet_datamap");
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
index 7e8a1c8..c138cc8 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java
@@ -219,24 +219,6 @@ public abstract class AbstractColumnDictionaryInfo implements DictionaryInfo {
}
/**
- * This method will set the sort order index of a dictionary column.
- * Sort order index if the index of dictionary values after they are sorted.
- *
- * @param sortOrderIndex
- */
- @Override public void setSortOrderIndex(List<Integer> sortOrderIndex) {
- }
-
- /**
- * This method will set the sort reverse index of a dictionary column.
- * Sort reverse index is the index of dictionary values before they are sorted.
- *
- * @param sortReverseOrderIndex
- */
- @Override public void setSortReverseOrderIndex(List<Integer> sortReverseOrderIndex) {
- }
-
- /**
* This method will find and return the dictionary value for a given surrogate key.
* Applicable scenarios:
* 1. Query final result preparation : While convert the final result which will
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
index ad1d201..e7c68a5 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
@@ -18,9 +18,7 @@
package org.apache.carbondata.core.cache.dictionary;
import java.nio.charset.Charset;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
@@ -301,17 +299,6 @@ public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo {
(Double.parseDouble(dictionaryVal)), (Double.parseDouble(memberVal)));
} else if (dataType == DataTypes.LONG) {
return Long.compare((Long.parseLong(dictionaryVal)), (Long.parseLong(memberVal)));
- } else if (dataType == DataTypes.BOOLEAN) {
- return Boolean.compare(
- (Boolean.parseBoolean(dictionaryVal)), (Boolean.parseBoolean(memberVal)));
- } else if (dataType == DataTypes.DATE || dataType == DataTypes.TIMESTAMP) {
- String format = CarbonUtil.getFormatFromProperty(dataType);
- SimpleDateFormat parser = new SimpleDateFormat(format);
- Date dateToStr;
- Date dictionaryDate;
- dateToStr = parser.parse(memberVal);
- dictionaryDate = parser.parse(dictionaryVal);
- return dictionaryDate.compareTo(dateToStr);
} else if (DataTypes.isDecimal(dataType)) {
java.math.BigDecimal javaDecValForDictVal = new java.math.BigDecimal(dictionaryVal);
java.math.BigDecimal javaDecValForMemberVal = new java.math.BigDecimal(memberVal);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
index 591fdf5..42d5b2e 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
@@ -111,4 +111,22 @@ public class ColumnReverseDictionaryInfo extends AbstractColumnDictionaryInfo {
private void createDictionaryByteArrayToSurrogateKeyMap(int initialMapSize) {
dictionaryByteArrayToSurrogateKeyMap = new ConcurrentHashMap<>(initialMapSize);
}
+
+ /**
+ * This method will set the sort order index of a dictionary column.
+ * Sort order index if the index of dictionary values after they are sorted.
+ *
+ * @param sortOrderIndex
+ */
+ @Override public void setSortOrderIndex(List<Integer> sortOrderIndex) {
+ }
+
+ /**
+ * This method will set the sort reverse index of a dictionary column.
+ * Sort reverse index is the index of dictionary values before they are sorted.
+ *
+ * @param sortReverseOrderIndex
+ */
+ @Override public void setSortReverseOrderIndex(List<Integer> sortReverseOrderIndex) {
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryBuilderException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryBuilderException.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryBuilderException.java
index 2bb984e..ed6394e 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryBuilderException.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryBuilderException.java
@@ -39,23 +39,5 @@ public class DictionaryBuilderException extends IOException {
super(msg);
this.msg = msg;
}
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-
- @Override public String toString() {
- return "DictionaryBuilderException: " + msg;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
index b54fb14..2450f85 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
@@ -25,7 +25,6 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -34,8 +33,6 @@ import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.path.CarbonTablePath;
/**
@@ -96,15 +93,6 @@ public class ManageDictionaryAndBTree {
* @param carbonTable
*/
public static void clearBTreeAndDictionaryLRUCache(CarbonTable carbonTable) {
- // clear Btree cache from LRU cache
- LoadMetadataDetails[] loadMetadataDetails =
- SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath());
- String[] segments = new String[loadMetadataDetails.length];
- int i = 0;
- for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
- segments[i++] = loadMetadataDetail.getLoadName();
- }
- invalidateBTreeCache(carbonTable.getAbsoluteTableIdentifier(), segments);
// clear dictionary cache from LRU cache
List<CarbonDimension> dimensions =
carbonTable.getDimensionByTableName(carbonTable.getTableName());
@@ -131,40 +119,4 @@ public class ManageDictionaryAndBTree {
dictCache = CacheProvider.getInstance().createCache(CacheType.FORWARD_DICTIONARY);
dictCache.invalidate(dictionaryColumnUniqueIdentifier);
}
-
- /**
- * This method will remove the BTree instances from LRU cache
- *
- * @param absoluteTableIdentifier
- * @param segments
- */
- public static void invalidateBTreeCache(AbsoluteTableIdentifier absoluteTableIdentifier,
- String[] segments) {
- Cache<Object, Object> driverBTreeCache =
- CacheProvider.getInstance().createCache(CacheType.DRIVER_BTREE);
- for (String segmentNo : segments) {
- TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier =
- new TableSegmentUniqueIdentifier(absoluteTableIdentifier, segmentNo);
- driverBTreeCache.invalidate(tableSegmentUniqueIdentifier);
- }
- }
-
- /**
- * This method will remove the BTree instances from LRU cache for all the segments
- *
- * @param carbonTable
- */
- public static void invalidateBTreeCache(CarbonTable carbonTable) {
- LoadMetadataDetails[] loadMetadataDetails =
- SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath());
- if (loadMetadataDetails.length > 0) {
- String[] segments = new String[loadMetadataDetails.length];
- int loadCounter = 0;
- for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
- segments[loadCounter++] = loadMetadataDetail.getLoadName();
- }
- invalidateBTreeCache(carbonTable.getAbsoluteTableIdentifier(), segments);
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/cache/update/BlockletLevelDeleteDeltaDataCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/update/BlockletLevelDeleteDeltaDataCache.java b/core/src/main/java/org/apache/carbondata/core/cache/update/BlockletLevelDeleteDeltaDataCache.java
deleted file mode 100644
index abad924..0000000
--- a/core/src/main/java/org/apache/carbondata/core/cache/update/BlockletLevelDeleteDeltaDataCache.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.cache.update;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-
-import org.roaringbitmap.RoaringBitmap;
-
-/**
- * This class maintains delete delta data cache of each blocklet along with the block timestamp
- */
-public class BlockletLevelDeleteDeltaDataCache {
- private Map<Integer, RoaringBitmap> deleteDelataDataCache =
- new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- private String timeStamp;
-
- public BlockletLevelDeleteDeltaDataCache(Map<Integer, Integer[]> deleteDeltaFileData,
- String timeStamp) {
- for (Map.Entry<Integer, Integer[]> entry : deleteDeltaFileData.entrySet()) {
- int[] dest = new int[entry.getValue().length];
- int i = 0;
- for (Integer val : entry.getValue()) {
- dest[i++] = val.intValue();
- }
- deleteDelataDataCache.put(entry.getKey(), RoaringBitmap.bitmapOf(dest));
- }
- this.timeStamp = timeStamp;
- }
-
- public boolean contains(int key, Integer pageId) {
- return deleteDelataDataCache.get(pageId).contains(key);
- }
-
- public String getCacheTimeStamp() {
- return timeStamp;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 7a64513..3e2843c 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -917,8 +917,6 @@ public final class CarbonCommonConstants {
*/
public static final String DATABASE_DEFAULT_NAME = "default";
- // tblproperties
- public static final String COLUMN_GROUPS = "column_groups";
public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
public static final String DICTIONARY_INCLUDE = "dictionary_include";
public static final String LONG_STRING_COLUMNS = "long_string_columns";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
index 168159b..086a1c0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
@@ -84,7 +84,7 @@ public abstract class DataMapProvider {
* This is called when user creates datamap, for example "CREATE DATAMAP dm ON TABLE mainTable"
* Implementation should initialize data for datamap, like creating data folders
*/
- public abstract void initData();
+ public void initData() { }
/**
* Opposite operation of {@link #initMeta(String)}.
@@ -111,13 +111,17 @@ public abstract class DataMapProvider {
/**
* Build the datamap incrementally by loading specified segment data
*/
- public abstract void incrementalBuild(String[] segmentIds) throws IOException;
+ public void incrementalBuild(String[] segmentIds) {
+ throw new UnsupportedOperationException();
+ }
/**
* Provide the datamap catalog instance or null if this datamap not required to rewrite
* the query.
*/
- public abstract DataMapCatalog createDataMapCatalog();
+ public DataMapCatalog createDataMapCatalog() {
+ return null;
+ }
public abstract DataMapFactory getDataMapFactory();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java b/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java
index 010c6c2..762d89c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DistributableDataMapFormat.java
@@ -29,10 +29,7 @@ import org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapper;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.util.ObjectSerializationUtil;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
@@ -45,8 +42,6 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
public class DistributableDataMapFormat extends FileInputFormat<Void, ExtendedBlocklet> implements
Serializable {
- private static final String FILTER_EXP = "mapreduce.input.distributed.datamap.filter";
-
private CarbonTable table;
private DataMapExprWrapper dataMapExprWrapper;
@@ -72,27 +67,6 @@ public class DistributableDataMapFormat extends FileInputFormat<Void, ExtendedBl
this.isJobToClearDataMaps = isJobToClearDataMaps;
}
- public boolean isJobToClearDataMaps() {
- return isJobToClearDataMaps;
- }
-
- public static void setFilterExp(Configuration configuration, FilterResolverIntf filterExp)
- throws IOException {
- if (filterExp != null) {
- String string = ObjectSerializationUtil.convertObjectToString(filterExp);
- configuration.set(FILTER_EXP, string);
- }
- }
-
- private static FilterResolverIntf getFilterExp(Configuration configuration) throws IOException {
- String filterString = configuration.get(FILTER_EXP);
- if (filterString != null) {
- Object toObject = ObjectSerializationUtil.convertStringToObject(filterString);
- return (FilterResolverIntf) toObject;
- }
- return null;
- }
-
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
List<DataMapDistributableWrapper> distributables =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapDistributableWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapDistributableWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapDistributableWrapper.java
index 9075032..22d2b90 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapDistributableWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapDistributableWrapper.java
@@ -42,10 +42,6 @@ public class DataMapDistributableWrapper extends InputSplit implements Serializa
return distributable;
}
- public void setDistributable(DataMapDistributable distributable) {
- this.distributable = distributable;
- }
-
@Override public long getLength() throws IOException, InterruptedException {
return distributable.getLength();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
deleted file mode 100644
index c642091..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
+++ /dev/null
@@ -1,481 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CarbonLRUCache;
-import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.block.SegmentTaskIndex;
-import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.mutate.UpdateVO;
-import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.ObjectSizeCalculator;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.core.util.path.CarbonTablePath.DataFileUtil;
-
-/**
- * Class to handle loading, unloading,clearing,storing of the table
- * blocks
- */
-public class SegmentTaskIndexStore
- implements Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper> {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(SegmentTaskIndexStore.class.getName());
- /**
- * CarbonLRU cache
- */
- protected CarbonLRUCache lruCache;
-
- /**
- * map of block info to lock object map, while loading the btree this will be filled
- * and removed after loading the tree for that particular block info, this will be useful
- * while loading the tree concurrently so only block level lock will be applied another
- * block can be loaded concurrently
- */
- private Map<String, Object> segmentLockMap;
-
- private Map<SegmentPropertiesWrapper, SegmentProperties> segmentProperties =
- new HashMap<SegmentPropertiesWrapper, SegmentProperties>();
-
- /**
- * constructor to initialize the SegmentTaskIndexStore
- *
- * @param lruCache
- */
- public SegmentTaskIndexStore(CarbonLRUCache lruCache) {
- this.lruCache = lruCache;
- segmentLockMap = new ConcurrentHashMap<String, Object>();
- }
-
- @Override
- public SegmentTaskIndexWrapper get(TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier)
- throws IOException {
- SegmentTaskIndexWrapper segmentTaskIndexWrapper = null;
- try {
- segmentTaskIndexWrapper =
- loadAndGetTaskIdToSegmentsMap(
- tableSegmentUniqueIdentifier.getSegmentToTableBlocksInfos(),
- CarbonTable.buildDummyTable("path"),
- tableSegmentUniqueIdentifier);
- } catch (IndexBuilderException e) {
- throw new IOException(e.getMessage(), e);
- } catch (Throwable e) {
- throw new IOException("Problem in loading segment block.", e);
- }
- return segmentTaskIndexWrapper;
- }
-
- @Override public List<SegmentTaskIndexWrapper> getAll(
- List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException {
- List<SegmentTaskIndexWrapper> segmentTaskIndexWrappers =
- new ArrayList<>(tableSegmentUniqueIdentifiers.size());
- try {
- for (TableSegmentUniqueIdentifier segmentUniqueIdentifier : tableSegmentUniqueIdentifiers) {
- segmentTaskIndexWrappers.add(get(segmentUniqueIdentifier));
- }
- } catch (Throwable e) {
- for (SegmentTaskIndexWrapper segmentTaskIndexWrapper : segmentTaskIndexWrappers) {
- segmentTaskIndexWrapper.clear();
- }
- throw new IOException("Problem in loading segment blocks.", e);
- }
- return segmentTaskIndexWrappers;
- }
-
- /**
- * returns the SegmentTaskIndexWrapper
- *
- * @param tableSegmentUniqueIdentifier
- * @return
- */
- @Override public SegmentTaskIndexWrapper getIfPresent(
- TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier) {
- SegmentTaskIndexWrapper segmentTaskIndexWrapper = (SegmentTaskIndexWrapper) lruCache
- .get(tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
- if (null != segmentTaskIndexWrapper) {
- segmentTaskIndexWrapper.incrementAccessCount();
- }
- return segmentTaskIndexWrapper;
- }
-
- /**
- * method invalidate the segment cache for segment
- *
- * @param tableSegmentUniqueIdentifier
- */
- @Override public void invalidate(TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier) {
- lruCache.remove(tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
- }
-
- @Override
- public void put(TableSegmentUniqueIdentifier key, SegmentTaskIndexWrapper value)
- throws IOException, MemoryException {
- throw new UnsupportedOperationException("Operation not supported");
- }
-
- /**
- * returns block timestamp value from the given task
- * @param taskKey
- * @param listOfUpdatedFactFiles
- * @return
- */
- private String getTimeStampValueFromBlock(String taskKey, List<String> listOfUpdatedFactFiles) {
- for (String blockName : listOfUpdatedFactFiles) {
- if (taskKey.equals(CarbonTablePath.DataFileUtil.getTaskNo(blockName))) {
- blockName = blockName.substring(blockName.lastIndexOf('-') + 1, blockName.lastIndexOf('.'));
- return blockName;
- }
- }
- return null;
- }
-
- /**
- * Below method will be used to load the segment of segments
- * One segment may have multiple task , so table segment will be loaded
- * based on task id and will return the map of taksId to table segment
- * map
- *
- * @param segmentToTableBlocksInfos segment id to block info
- * @param table table handle
- * @return map of taks id to segment mapping
- * @throws IOException
- */
- private SegmentTaskIndexWrapper loadAndGetTaskIdToSegmentsMap(
- Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos,
- CarbonTable table,
- TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier) throws IOException {
- // task id to segment map
- Iterator<Map.Entry<String, List<TableBlockInfo>>> iteratorOverSegmentBlocksInfos =
- segmentToTableBlocksInfos.entrySet().iterator();
- Map<TaskBucketHolder, AbstractIndex> taskIdToSegmentIndexMap = null;
- SegmentTaskIndexWrapper segmentTaskIndexWrapper = null;
- SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(table);
- String segmentId = null;
- TaskBucketHolder taskBucketHolder = null;
- try {
- while (iteratorOverSegmentBlocksInfos.hasNext()) {
- // Initialize the UpdateVO to Null for each segment.
- UpdateVO updateVO = null;
- // segment id to table block mapping
- Map.Entry<String, List<TableBlockInfo>> next = iteratorOverSegmentBlocksInfos.next();
- // group task id to table block info mapping for the segment
- Map<TaskBucketHolder, List<TableBlockInfo>> taskIdToTableBlockInfoMap =
- mappedAndGetTaskIdToTableBlockInfo(segmentToTableBlocksInfos);
- segmentId = next.getKey();
- // updateVO is only required when Updates Or Delete performed on the Table.
- if (updateStatusManager.getUpdateStatusDetails().length != 0) {
- // get the existing map of task id to table segment map
- updateVO = updateStatusManager.getInvalidTimestampRange(segmentId);
- }
- // check if segment is already loaded, if segment is already loaded
- //no need to load the segment block
- String lruCacheKey = tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier();
- segmentTaskIndexWrapper = (SegmentTaskIndexWrapper) lruCache.get(lruCacheKey);
- if ((segmentTaskIndexWrapper == null) || ((null != updateVO)
- && (tableSegmentUniqueIdentifier.isSegmentUpdated()))) {
- // get the segment loader lock object this is to avoid
- // same segment is getting loaded multiple times
- // in case of concurrent query
- Object segmentLoderLockObject = segmentLockMap.get(lruCacheKey);
- if (null == segmentLoderLockObject) {
- segmentLoderLockObject = addAndGetSegmentLock(lruCacheKey);
- }
- // acquire lock to lod the segment
- synchronized (segmentLoderLockObject) {
- segmentTaskIndexWrapper = (SegmentTaskIndexWrapper) lruCache.get(lruCacheKey);
- if ((null == segmentTaskIndexWrapper) || ((null != updateVO)
- && (tableSegmentUniqueIdentifier.isSegmentUpdated()))) {
- // if the segment is updated then get the existing block task id map details
- // so that the same can be updated after loading the btree.
- if (tableSegmentUniqueIdentifier.isSegmentUpdated()
- && null != segmentTaskIndexWrapper) {
- taskIdToSegmentIndexMap = segmentTaskIndexWrapper.getTaskIdToTableSegmentMap();
- } else {
- // creating a map of take if to table segment
- taskIdToSegmentIndexMap = new HashMap<TaskBucketHolder, AbstractIndex>();
- segmentTaskIndexWrapper = new SegmentTaskIndexWrapper(taskIdToSegmentIndexMap);
- segmentTaskIndexWrapper.incrementAccessCount();
- }
- Iterator<Map.Entry<TaskBucketHolder, List<TableBlockInfo>>> iterator =
- taskIdToTableBlockInfoMap.entrySet().iterator();
- AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
- long requiredSize =
- calculateRequiredSize(taskIdToTableBlockInfoMap, absoluteTableIdentifier);
- segmentTaskIndexWrapper.setMemorySize(requiredSize);
- boolean canAddToLruCache =
- lruCache.tryPut(lruCacheKey, requiredSize);
- if (canAddToLruCache) {
- while (iterator.hasNext()) {
- Map.Entry<TaskBucketHolder, List<TableBlockInfo>> taskToBlockInfoList =
- iterator.next();
- taskBucketHolder = taskToBlockInfoList.getKey();
- taskIdToSegmentIndexMap.put(taskBucketHolder,
- loadBlocks(taskBucketHolder, taskToBlockInfoList.getValue(),
- absoluteTableIdentifier));
- }
- long updatedRequiredSize =
- ObjectSizeCalculator.estimate(segmentTaskIndexWrapper, requiredSize);
- // update the actual size of object
- segmentTaskIndexWrapper.setMemorySize(updatedRequiredSize);
- if (!lruCache.put(lruCacheKey, segmentTaskIndexWrapper, updatedRequiredSize)) {
- throw new IndexBuilderException(
- "Can not load the segment. No Enough space available.");
- }
-
- } else {
- throw new IndexBuilderException(
- "Can not load the segment. No Enough space available.");
- }
-
- // Refresh the Timestamp for those tables which underwent through IUD Operations.
- if (null != updateVO) {
- // set the latest timestamp.
- segmentTaskIndexWrapper
- .setRefreshedTimeStamp(updateVO.getCreatedOrUpdatedTimeStamp());
- } else {
- segmentTaskIndexWrapper.setRefreshedTimeStamp(0L);
- }
- // tableSegmentMapTemp.put(next.getKey(), taskIdToSegmentIndexMap);
- // removing from segment lock map as once segment is loaded
- // if concurrent query is coming for same segment
- // it will wait on the lock so after this segment will be already
- // loaded so lock is not required, that is why removing the
- // the lock object as it wont be useful
- segmentLockMap.remove(lruCacheKey);
- } else {
- segmentTaskIndexWrapper.incrementAccessCount();
- }
- }
- } else {
- segmentTaskIndexWrapper.incrementAccessCount();
- }
- }
- } catch (IndexBuilderException e) {
- LOGGER.error("Problem while loading the segment");
- throw e;
- }
- return segmentTaskIndexWrapper;
- }
-
- private long calculateRequiredSize(
- Map<TaskBucketHolder, List<TableBlockInfo>> taskIdToTableBlockInfoMap,
- AbsoluteTableIdentifier absoluteTableIdentifier) {
- Iterator<Map.Entry<TaskBucketHolder, List<TableBlockInfo>>> iterator =
- taskIdToTableBlockInfoMap.entrySet().iterator();
- TaskBucketHolder taskBucketHolder;
- long driverBTreeSize = 0;
- while (iterator.hasNext()) {
- Map.Entry<TaskBucketHolder, List<TableBlockInfo>> taskToBlockInfoList = iterator.next();
- taskBucketHolder = taskToBlockInfoList.getKey();
- driverBTreeSize += CarbonUtil
- .calculateDriverBTreeSize(taskBucketHolder.taskNo, taskBucketHolder.bucketNumber,
- taskToBlockInfoList.getValue(), absoluteTableIdentifier);
- }
- return driverBTreeSize;
- }
-
- /**
- * Below method will be used to get the task id to all the table block info belongs to
- * that task id mapping
- *
- * @param segmentToTableBlocksInfos segment if to table blocks info map
- * @return task id to table block info mapping
- */
- private Map<TaskBucketHolder, List<TableBlockInfo>> mappedAndGetTaskIdToTableBlockInfo(
- Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos) {
- Map<TaskBucketHolder, List<TableBlockInfo>> taskIdToTableBlockInfoMap =
- new ConcurrentHashMap<>();
- Iterator<Entry<String, List<TableBlockInfo>>> iterator =
- segmentToTableBlocksInfos.entrySet().iterator();
- while (iterator.hasNext()) {
- Entry<String, List<TableBlockInfo>> next = iterator.next();
- List<TableBlockInfo> value = next.getValue();
- for (TableBlockInfo blockInfo : value) {
- String taskNo = DataFileUtil.getTaskNo(blockInfo.getFilePath());
- String bucketNo = DataFileUtil.getBucketNo(blockInfo.getFilePath());
- TaskBucketHolder bucketHolder = new TaskBucketHolder(taskNo, bucketNo);
- List<TableBlockInfo> list = taskIdToTableBlockInfoMap.get(bucketHolder);
- if (null == list) {
- list = new ArrayList<TableBlockInfo>();
- taskIdToTableBlockInfoMap.put(bucketHolder, list);
- }
- list.add(blockInfo);
- }
-
- }
- return taskIdToTableBlockInfoMap;
- }
-
- /**
- * Below method will be used to get the segment level lock object
- *
- * @param segmentId
- * @return lock object
- */
- private synchronized Object addAndGetSegmentLock(String segmentId) {
- // get the segment lock object if it is present then return
- // otherwise add the new lock and return
- Object segmentLoderLockObject = segmentLockMap.get(segmentId);
- if (null == segmentLoderLockObject) {
- segmentLoderLockObject = new Object();
- segmentLockMap.put(segmentId, segmentLoderLockObject);
- }
- return segmentLoderLockObject;
- }
-
- /**
- * Below method will be used to load the blocks
- *
- * @param tableBlockInfoList
- * @return loaded segment
- * @throws IOException
- */
- private AbstractIndex loadBlocks(TaskBucketHolder taskBucketHolder,
- List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier tableIdentifier)
- throws IOException {
- // all the block of one task id will be loaded together
- // so creating a list which will have all the data file meta data to of one task
- List<DataFileFooter> footerList = CarbonUtil
- .readCarbonIndexFile(taskBucketHolder.taskNo, taskBucketHolder.bucketNumber,
- tableBlockInfoList, tableIdentifier);
-
- // Reuse SegmentProperties object if tableIdentifier, columnsInTable and columnCardinality are
- // the same.
- List<ColumnSchema> columnsInTable = footerList.get(0).getColumnInTable();
- int[] columnCardinality = footerList.get(0).getSegmentInfo().getColumnCardinality();
- SegmentPropertiesWrapper segmentPropertiesWrapper =
- new SegmentPropertiesWrapper(tableIdentifier, columnsInTable, columnCardinality);
- SegmentProperties segmentProperties;
- if (this.segmentProperties.containsKey(segmentPropertiesWrapper)) {
- segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper);
- } else {
- // create a metadata details
- // this will be useful in query handling
- // all the data file metadata will have common segment properties we
- // can use first one to get create the segment properties
- segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
- this.segmentProperties.put(segmentPropertiesWrapper, segmentProperties);
- }
-
- AbstractIndex segment = new SegmentTaskIndex(segmentProperties);
- // file path of only first block is passed as it all table block info path of
- // same task id will be same
- segment.buildIndex(footerList);
- return segment;
- }
-
- /**
- * The method clears the access count of table segments
- *
- * @param tableSegmentUniqueIdentifiers
- */
- @Override
- public void clearAccessCount(List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers) {
- for (TableSegmentUniqueIdentifier segmentUniqueIdentifier : tableSegmentUniqueIdentifiers) {
- SegmentTaskIndexWrapper cacheable = (SegmentTaskIndexWrapper) lruCache
- .get(segmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
- cacheable.clear();
- }
- }
-
- public static class TaskBucketHolder implements Serializable {
-
- public String taskNo;
-
- public String bucketNumber;
-
- public TaskBucketHolder(String taskNo, String bucketNumber) {
- this.taskNo = taskNo;
- this.bucketNumber = bucketNumber;
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TaskBucketHolder that = (TaskBucketHolder) o;
-
- if (taskNo != null ? !taskNo.equals(that.taskNo) : that.taskNo != null) return false;
- return bucketNumber != null ?
- bucketNumber.equals(that.bucketNumber) :
- that.bucketNumber == null;
-
- }
-
- @Override public int hashCode() {
- int result = taskNo != null ? taskNo.hashCode() : 0;
- result = 31 * result + (bucketNumber != null ? bucketNumber.hashCode() : 0);
- return result;
- }
- }
-
- /**
- * This class wraps tableIdentifier, columnsInTable and columnCardinality as a key to determine
- * whether the SegmentProperties object can be reused.
- */
- public static class SegmentPropertiesWrapper {
- private AbsoluteTableIdentifier tableIdentifier;
- private List<ColumnSchema> columnsInTable;
- private int[] columnCardinality;
-
- public SegmentPropertiesWrapper(AbsoluteTableIdentifier tableIdentifier,
- List<ColumnSchema> columnsInTable, int[] columnCardinality) {
- this.tableIdentifier = tableIdentifier;
- this.columnsInTable = columnsInTable;
- this.columnCardinality = columnCardinality;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof SegmentPropertiesWrapper)) {
- return false;
- }
- SegmentPropertiesWrapper other = (SegmentPropertiesWrapper) obj;
- return tableIdentifier.equals(other.tableIdentifier)
- && columnsInTable.equals(other.columnsInTable)
- && Arrays.equals(columnCardinality, other.columnCardinality);
- }
-
- @Override
- public int hashCode() {
- return tableIdentifier.hashCode()
- + columnsInTable.hashCode() + Arrays.hashCode(columnCardinality);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
index 888689b..18b856b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
@@ -17,11 +17,7 @@
package org.apache.carbondata.core.datastore;
-import java.util.List;
-import java.util.Map;
-
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
@@ -34,13 +30,7 @@ public class TableSegmentUniqueIdentifier {
*/
private AbsoluteTableIdentifier absoluteTableIdentifier;
- /**
- * segment to tableBlockInfo map
- */
- Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos;
-
private String segmentId;
- private boolean isSegmentUpdated;
/**
* Constructor to initialize the class instance
@@ -53,42 +43,6 @@ public class TableSegmentUniqueIdentifier {
this.segmentId = segmentId;
}
- public TableSegmentUniqueIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier,
- Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos, String segmentId) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
- this.segmentToTableBlocksInfos = segmentToTableBlocksInfos;
- this.segmentId = segmentId;
- }
-
- /**
- * returns AbsoluteTableIdentifier
- * @return
- */
- public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
- return absoluteTableIdentifier;
- }
-
- public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
- }
-
- /**
- * returns the segment to tableBlockInfo map
- * @return
- */
- public Map<String, List<TableBlockInfo>> getSegmentToTableBlocksInfos() {
- return segmentToTableBlocksInfos;
- }
-
- /**
- * set the segment to tableBlockInfo map
- * @param segmentToTableBlocksInfos
- */
- public void setSegmentToTableBlocksInfos(
- Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos) {
- this.segmentToTableBlocksInfos = segmentToTableBlocksInfos;
- }
-
public String getSegmentId() {
return segmentId;
}
@@ -105,13 +59,6 @@ public class TableSegmentUniqueIdentifier {
+ carbonTableIdentifier.getTableName() + CarbonCommonConstants.UNDERSCORE
+ carbonTableIdentifier.getTableId() + CarbonCommonConstants.FILE_SEPARATOR + segmentId;
}
- public void setIsSegmentUpdated(boolean isSegmentUpdated) {
- this.isSegmentUpdated = isSegmentUpdated;
- }
-
- public boolean isSegmentUpdated() {
- return isSegmentUpdated;
- }
/**
* equals method to compare two objects having same
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
index 93da81e..4d30cb0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
@@ -50,12 +50,8 @@ public class TableSpec {
// first calculate total number of columnar field considering column group and complex column
numSimpleDimensions = 0;
for (CarbonDimension dimension : dimensions) {
- if (dimension.isColumnar()) {
- if (!dimension.isComplex()) {
- numSimpleDimensions++;
- }
- } else {
- throw new UnsupportedOperationException("column group is not supported");
+ if (!dimension.isComplex()) {
+ numSimpleDimensions++;
}
}
dimensionSpec = new DimensionSpec[dimensions.size()];
@@ -68,24 +64,22 @@ public class TableSpec {
int dimIndex = 0;
for (int i = 0; i < dimensions.size(); i++) {
CarbonDimension dimension = dimensions.get(i);
- if (dimension.isColumnar()) {
- if (dimension.isComplex()) {
- DimensionSpec spec = new DimensionSpec(ColumnType.COMPLEX, dimension);
- dimensionSpec[dimIndex++] = spec;
- } else if (dimension.getDataType() == DataTypes.TIMESTAMP && !dimension
- .isDirectDictionaryEncoding()) {
- DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension);
- dimensionSpec[dimIndex++] = spec;
- } else if (dimension.isDirectDictionaryEncoding()) {
- DimensionSpec spec = new DimensionSpec(ColumnType.DIRECT_DICTIONARY, dimension);
- dimensionSpec[dimIndex++] = spec;
- } else if (dimension.isGlobalDictionaryEncoding()) {
- DimensionSpec spec = new DimensionSpec(ColumnType.GLOBAL_DICTIONARY, dimension);
- dimensionSpec[dimIndex++] = spec;
- } else {
- DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension);
- dimensionSpec[dimIndex++] = spec;
- }
+ if (dimension.isComplex()) {
+ DimensionSpec spec = new DimensionSpec(ColumnType.COMPLEX, dimension);
+ dimensionSpec[dimIndex++] = spec;
+ } else if (dimension.getDataType() == DataTypes.TIMESTAMP && !dimension
+ .isDirectDictionaryEncoding()) {
+ DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension);
+ dimensionSpec[dimIndex++] = spec;
+ } else if (dimension.isDirectDictionaryEncoding()) {
+ DimensionSpec spec = new DimensionSpec(ColumnType.DIRECT_DICTIONARY, dimension);
+ dimensionSpec[dimIndex++] = spec;
+ } else if (dimension.isGlobalDictionaryEncoding()) {
+ DimensionSpec spec = new DimensionSpec(ColumnType.GLOBAL_DICTIONARY, dimension);
+ dimensionSpec[dimIndex++] = spec;
+ } else {
+ DimensionSpec spec = new DimensionSpec(ColumnType.PLAIN_VALUE, dimension);
+ dimensionSpec[dimIndex++] = spec;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/block/AbstractIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/AbstractIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/AbstractIndex.java
index 4d0e56d..7fbef8a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/AbstractIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/AbstractIndex.java
@@ -38,11 +38,6 @@ public abstract class AbstractIndex implements Cacheable {
protected DataRefNode dataRefNode;
/**
- * total number of row present in the block
- */
- protected long totalNumberOfRows;
-
- /**
* atomic integer to maintain the access count for a column access
*/
protected AtomicInteger accessCount = new AtomicInteger();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index f652381..d507937 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -27,7 +27,6 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator;
@@ -126,19 +125,6 @@ public class SegmentProperties {
private int[] eachComplexDimColumnValueSize;
/**
- * below mapping will have mapping of the column group to dimensions ordinal
- * for example if 3 dimension present in the columngroupid 0 and its ordinal in
- * 2,3,4 then map will contain 0,{2,3,4}
- */
- private Map<Integer, KeyGenerator> columnGroupAndItsKeygenartor;
-
- /**
- * column group key generator dimension index will not be same as dimension ordinal
- * This will have mapping with ordinal and keygenerator or mdkey index
- */
- private Map<Integer, Map<Integer, Integer>> columnGroupOrdinalToMdkeymapping;
-
- /**
* this will be used to split the fixed length key
* this will all the information about how key was created
* and how to split the key based on group
@@ -153,11 +139,6 @@ public class SegmentProperties {
*/
private int numberOfNoDictionaryDimension;
- /**
- * column group model
- */
- private ColumnGroupModel colGroupModel;
-
private int numberOfSortColumns = 0;
private int numberOfNoDictSortColumns = 0;
@@ -176,50 +157,12 @@ public class SegmentProperties {
new HashMap<Integer, Set<Integer>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
measuresOrdinalToChunkMapping =
new HashMap<Integer, Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- intialiseColGroups();
fillOrdinalToBlockMappingForDimension();
fillOrdinalToChunkIndexMappingForMeasureColumns();
- fillColumnGroupAndItsCardinality(columnCardinality);
fillKeyGeneratorDetails();
}
/**
- * it fills column groups
- * e.g {{1},{2,3,4},{5},{6},{7,8,9}}
- *
- */
- private void intialiseColGroups() {
- List<List<Integer>> colGrpList = new ArrayList<List<Integer>>();
- List<Integer> group = new ArrayList<Integer>();
- for (int i = 0; i < dimensions.size(); i++) {
- CarbonDimension dimension = dimensions.get(i);
- if (!dimension.hasEncoding(Encoding.DICTIONARY)) {
- continue;
- }
- group.add(dimension.getOrdinal());
- if (i < dimensions.size() - 1) {
- int currGroupOrdinal = dimension.columnGroupId();
- int nextGroupOrdinal = dimensions.get(i + 1).columnGroupId();
- if (!(currGroupOrdinal == nextGroupOrdinal && currGroupOrdinal != -1)) {
- colGrpList.add(group);
- group = new ArrayList<Integer>();
- }
- } else {
- colGrpList.add(group);
- }
-
- }
- int[][] colGroups = new int[colGrpList.size()][];
- for (int i = 0; i < colGroups.length; i++) {
- colGroups[i] = new int[colGrpList.get(i).size()];
- for (int j = 0; j < colGroups[i].length; j++) {
- colGroups[i][j] = colGrpList.get(i).get(j);
- }
- }
- this.colGroupModel = CarbonUtil.getColGroupModel(colGroups);
- }
-
- /**
* below method is to fill the dimension and its mapping to file blocks all
* the column will point to same column group
*/
@@ -227,16 +170,10 @@ public class SegmentProperties {
int blockOrdinal = -1;
CarbonDimension dimension = null;
int index = 0;
- int prvcolumnGroupId = -1;
while (index < dimensions.size()) {
dimension = dimensions.get(index);
- // if column id is same as previous one then block index will be
- // same
- if (dimension.isColumnar() || dimension.columnGroupId() != prvcolumnGroupId) {
- blockOrdinal++;
- }
+ blockOrdinal++;
dimensionOrdinalToChunkMapping.put(dimension.getOrdinal(), blockOrdinal);
- prvcolumnGroupId = dimension.columnGroupId();
index++;
}
index = 0;
@@ -333,9 +270,6 @@ public class SegmentProperties {
// to store the position of dimension in surrogate key array which is
// participating in mdkey
int keyOrdinal = 0;
- int previousColumnGroup = -1;
- // to store the ordinal of the column group ordinal
- int columnGroupOrdinal = 0;
int counter = 0;
int complexTypeOrdinal = -1;
while (counter < columnsInTable.size()) {
@@ -350,36 +284,17 @@ public class SegmentProperties {
if (columnSchema.isSortColumn()) {
this.numberOfSortColumns++;
}
- if (columnSchema.isColumnar()) {
- // if it is a columnar dimension participated in mdkey then added
- // key ordinal and dimension ordinal
- carbonDimension =
- new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, -1, -1);
- } else {
- // if not columnnar then it is a column group dimension
-
- // below code to handle first dimension of the column group
- // in this case ordinal of the column group will be 0
- if (previousColumnGroup != columnSchema.getColumnGroupId()) {
- columnGroupOrdinal = 0;
- carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++,
- columnGroupOrdinal++, -1);
- }
- // if previous dimension column group id is same as current then
- // then its belongs to same row group
- else {
- carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++,
- columnGroupOrdinal++, -1);
- }
- previousColumnGroup = columnSchema.getColumnGroupId();
- }
+ // if it is a columnar dimension participated in mdkey then added
+ // key ordinal and dimension ordinal
+ carbonDimension =
+ new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, -1);
}
// as complex type will be stored at last so once complex type started all the dimension
// will be added to complex type
else if (isComplexDimensionStarted || columnSchema.getDataType().isComplexType()) {
cardinalityIndexForComplexDimensionColumn.add(tableOrdinal);
carbonDimension =
- new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, ++complexTypeOrdinal);
+ new CarbonDimension(columnSchema, dimensonOrdinal++, -1, ++complexTypeOrdinal);
carbonDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
complexDimensions.add(carbonDimension);
isComplexDimensionStarted = true;
@@ -396,7 +311,7 @@ public class SegmentProperties {
continue;
} else {
// for no dictionary dimension
- carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, -1);
+ carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1);
numberOfNoDictionaryDimension++;
if (columnSchema.isSortColumn()) {
this.numberOfSortColumns++;
@@ -444,8 +359,7 @@ public class SegmentProperties {
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
- complexDimensionOrdinal++);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, -1, complexDimensionOrdinal++);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
parentDimension.getListOfChildDimensions().add(complexDimension);
dimensionOrdinal =
@@ -453,8 +367,7 @@ public class SegmentProperties {
listOfColumns, complexDimension, complexDimensionOrdinal);
} else {
parentDimension.getListOfChildDimensions().add(
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
- complexDimensionOrdinal++));
+ new CarbonDimension(columnSchema, dimensionOrdinal++, -1, complexDimensionOrdinal++));
}
}
}
@@ -493,7 +406,6 @@ public class SegmentProperties {
new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<Boolean> isDictionaryColumn =
new ArrayList<Boolean>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- int prvcolumnGroupId = -1;
int counter = 0;
while (counter < dimensions.size()) {
CarbonDimension carbonDimension = dimensions.get(counter);
@@ -503,23 +415,8 @@ public class SegmentProperties {
counter++;
continue;
}
- // columnar column is stored individually
- // so add one
- if (carbonDimension.isColumnar()) {
- dimensionPartitionList.add(1);
- isDictionaryColumn.add(true);
- }
- // if in a group then need to add how many columns a selected in
- // group
- if (!carbonDimension.isColumnar() && carbonDimension.columnGroupId() == prvcolumnGroupId) {
- // incrementing the previous value of the list as it is in same column group
- dimensionPartitionList.set(dimensionPartitionList.size() - 1,
- dimensionPartitionList.get(dimensionPartitionList.size() - 1) + 1);
- } else if (!carbonDimension.isColumnar()) {
- dimensionPartitionList.add(1);
- isDictionaryColumn.add(true);
- }
- prvcolumnGroupId = carbonDimension.columnGroupId();
+ dimensionPartitionList.add(1);
+ isDictionaryColumn.add(true);
counter++;
}
// get the partitioner
@@ -570,73 +467,6 @@ public class SegmentProperties {
}
/**
- * Below method will be used to create a mapping of column group and its column cardinality this
- * mapping will have column group id to cardinality of the dimension present in
- * the column group.This mapping will be used during query execution, to create
- * a mask key for the column group dimension which will be used in aggregation
- * and filter query as column group dimension will be stored at the bit level
- */
- private void fillColumnGroupAndItsCardinality(int[] cardinality) {
- // mapping of the column group and its ordinal
- Map<Integer, List<Integer>> columnGroupAndOrdinalMapping =
- new HashMap<Integer, List<Integer>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- // to store a column group
- List<Integer> currentColumnGroup = null;
- // current index
- int index = 0;
- // previous column group to check all the column of column id has bee selected
- int prvColumnGroupId = -1;
- while (index < dimensions.size()) {
- // if dimension group id is not zero and it is same as the previous
- // column id
- // then we need to add ordinal of that column as it belongs to same
- // column group
- if (!dimensions.get(index).isColumnar()
- && dimensions.get(index).columnGroupId() == prvColumnGroupId
- && null != currentColumnGroup) {
- currentColumnGroup.add(index);
- }
- // if column is not a columnar then new column group has come
- // so we need to create a list of new column id group and add the
- // ordinal
- else if (!dimensions.get(index).isColumnar()) {
- currentColumnGroup = new ArrayList<Integer>();
- columnGroupAndOrdinalMapping.put(dimensions.get(index).columnGroupId(), currentColumnGroup);
- currentColumnGroup.add(index);
- }
- // update the column id every time,this is required to group the
- // columns
- // of the same column group
- prvColumnGroupId = dimensions.get(index).columnGroupId();
- index++;
- }
- // Initializing the map
- this.columnGroupAndItsKeygenartor =
- new HashMap<Integer, KeyGenerator>(columnGroupAndOrdinalMapping.size());
- this.columnGroupOrdinalToMdkeymapping = new HashMap<>(columnGroupAndOrdinalMapping.size());
- int[] columnGroupCardinality = null;
- index = 0;
- Iterator<Entry<Integer, List<Integer>>> iterator =
- columnGroupAndOrdinalMapping.entrySet().iterator();
- while (iterator.hasNext()) {
- Entry<Integer, List<Integer>> next = iterator.next();
- List<Integer> currentGroupOrdinal = next.getValue();
- Map<Integer, Integer> colGrpOrdinalMdkeyMapping = new HashMap<>(currentGroupOrdinal.size());
- // create the cardinality array
- columnGroupCardinality = new int[currentGroupOrdinal.size()];
- for (int i = 0; i < columnGroupCardinality.length; i++) {
- // fill the cardinality
- columnGroupCardinality[i] = cardinality[currentGroupOrdinal.get(i)];
- colGrpOrdinalMdkeyMapping.put(currentGroupOrdinal.get(i), i);
- }
- this.columnGroupAndItsKeygenartor.put(next.getKey(), new MultiDimKeyVarLengthGenerator(
- CarbonUtil.getDimensionBitLength(columnGroupCardinality,
- new int[] { columnGroupCardinality.length })));
- this.columnGroupOrdinalToMdkeymapping.put(next.getKey(), colGrpOrdinalMdkeyMapping);
- }
- }
-
- /**
* Below method is to get the value of each dimension column. As this method
* will be used only once so we can merge both the dimension and complex
* dimension array. Complex dimension will be store at last so first copy
@@ -764,13 +594,6 @@ public class SegmentProperties {
}
/**
- * @return the columnGroupAndItsKeygenartor
- */
- public Map<Integer, KeyGenerator> getColumnGroupAndItsKeygenartor() {
- return columnGroupAndItsKeygenartor;
- }
-
- /**
* @return the numberOfNoDictionaryDimension
*/
public int getNumberOfNoDictionaryDimension() {
@@ -778,31 +601,6 @@ public class SegmentProperties {
}
/**
- * @return
- */
- public int[][] getColumnGroups() {
- return colGroupModel.getColumnGroup();
- }
-
- /**
- * @return colGroupModel
- */
- public ColumnGroupModel getColumnGroupModel() {
- return this.colGroupModel;
- }
-
- /**
- * get mdkey ordinal for given dimension ordinal of given column group
- *
- * @param colGrpId
- * @param ordinal
- * @return mdkeyordinal
- */
- public int getColumnGroupMdKeyOrdinal(int colGrpId, int ordinal) {
- return columnGroupOrdinalToMdkeymapping.get(colGrpId).get(ordinal);
- }
-
- /**
* @param blockIndex
* @return It returns all dimension present in given block index
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
deleted file mode 100644
index f577ede..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.block;
-
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
-import org.apache.carbondata.core.datastore.BtreeBuilder;
-import org.apache.carbondata.core.datastore.impl.btree.BlockBTreeBuilder;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-
-/**
- * Class which is responsible for loading the b+ tree block. This class will
- * persist all the detail of a table segment
- */
-public class SegmentTaskIndex extends AbstractIndex {
-
- public SegmentTaskIndex(SegmentProperties segmentProperties) {
- this.segmentProperties = segmentProperties;
- }
-
- /**
- * Below method is store the blocks in some data structure
- *
- */
- public void buildIndex(List<DataFileFooter> footerList) {
- // create a segment builder info
- // in case of segment create we do not need any file path and each column value size
- // as Btree will be build as per min max and start key
- BTreeBuilderInfo btreeBuilderInfo = new BTreeBuilderInfo(footerList, null);
- BtreeBuilder blocksBuilder = new BlockBTreeBuilder();
- // load the metadata
- blocksBuilder.build(btreeBuilderInfo);
- dataRefNode = blocksBuilder.get();
- for (DataFileFooter footer : footerList) {
- totalNumberOfRows += footer.getNumberOfRows();
- }
- }
-}
[4/8] carbondata git commit: [CARBONDATA-2720] Remove dead code
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
index 38a7222..5391861 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
@@ -106,6 +106,40 @@ public class GreaterThanExpressionUnitTest {
}
+ @Test public void testEvaluateForGreaterThanExpressionWithShortDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("id", DataTypes.SHORT);
+ left.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 170 };
+ Short[] row1 = { 70 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Short getShort() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 70;
+
+ } else {
+ return 170;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
@Test public void testEvaluateForGreaterThanExpressionWithDoubleDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DOUBLE);
@@ -217,6 +251,51 @@ public class GreaterThanExpressionUnitTest {
}
}
+ @Test public void testEvaluateForGreaterThanExpressionWithTimestampDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("timestamp", DataTypes.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("timestamp", DataTypes.TIMESTAMP);
+ right.setColIndex(1);
+
+ greaterThanExpression = new GreaterThanExpression(left, right);
+
+ RowImpl value = new RowImpl();
+
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+
+ Date date1 = dateFormat.parse("24/09/2007");
+ long time1 = date1.getTime();
+ Timestamp[] row1 = { new Timestamp(time1) };
+
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getTime() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1190505600L;
+ } else {
+ return 1190592000L;
+ }
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
@Test public void testEvaluateForGreaterThanExpressionWithLongDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.LONG);
@@ -247,6 +326,36 @@ public class GreaterThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForGreaterThanExpressionWithLongDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.LONG);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataTypes.LONG);
+ left.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 1234567654321L };
+ Long[] row1 = { 123456765432234L };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getLong() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1234567654321L;
+ } else {
+ return 123456765432234L;
+ }
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testEvaluateForGreaterThanExpressionWithDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
@@ -277,6 +386,36 @@ public class GreaterThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForGreaterThanExpressionWithDecimalDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
+ left.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ BigDecimal[] row = new BigDecimal[] { new BigDecimal(12345.0) };
+ BigDecimal[] row1 = new BigDecimal[] { new BigDecimal(123451245.0) };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public BigDecimal getDecimal() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return new BigDecimal(12345.0);
+ } else {
+ return new BigDecimal(123451245.0);
+ }
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testForGreaterThanExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
@@ -316,6 +455,38 @@ public class GreaterThanExpressionUnitTest {
}
+ @Test public void testEvaluateForGreaterThanExpressionWithIsNullReturnTrue1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
+ right.setColIndex(0);
+ greaterThanExpression = new GreaterThanExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public boolean isNull() {
+ if (isFirst) {
+ isFirst = false;
+ return false;
+ }
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
@Test public void testEvaluateForGreaterThanExpressionWithLeftAndRightDifferentDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("name", DataTypes.STRING);
@@ -348,6 +519,38 @@ public class GreaterThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForGreaterThanExpressionWithLeftAndRightDifferentDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataTypes.INT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataTypes.STRING);
+ right.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "String1" };
+ Integer[] row1 = { 14 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 15;
+ } else {
+ return 14;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
@Test public void testForGreaterThanExpressionWithGetString() throws Exception {
ColumnExpression right = new ColumnExpression("right_name", DataTypes.STRING);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
index d3c8cf4..14594e7 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
@@ -109,6 +109,40 @@ public class LessThanEqualToExpressionUnitTest {
}
+ @Test public void testEvaluateForLessThanEqualToExpressionWithShortDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("id", DataTypes.SHORT);
+ left.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 1550 };
+ Short[] row1 = { 3365 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Short getShort() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 3365;
+
+ } else {
+ return 1550;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
@Test public void testEvaluateForLessThanEqualToExpressionWithDoubleDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DOUBLE);
@@ -220,6 +254,51 @@ public class LessThanEqualToExpressionUnitTest {
}
}
+ @Test public void testEvaluateForLessThanEqualToExpressionWithTimestampDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("timestamp", DataTypes.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("timestamp", DataTypes.TIMESTAMP);
+ right.setColIndex(1);
+
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+
+ RowImpl value = new RowImpl();
+
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+
+ Date date1 = dateFormat.parse("24/09/2007");
+ long time1 = date1.getTime();
+ Timestamp[] row1 = { new Timestamp(time1) };
+
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getTime() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1190592000L;
+ } else {
+ return 1190505600L;
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
@Test public void testEvaluateForLessThanEqualToExpressionWithLongDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("right_contact", DataTypes.LONG);
@@ -280,6 +359,38 @@ public class LessThanEqualToExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForLessThanEqualToExpressionWithDecimalDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right =
+ new ColumnExpression("right_contact", DataTypes.createDefaultDecimalType());
+ right.setColIndex(0);
+ ColumnExpression left =
+ new ColumnExpression("left_contact", DataTypes.createDefaultDecimalType());
+ left.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ BigDecimal[] row = new BigDecimal[] { new BigDecimal(46851.2) };
+ BigDecimal[] row1 = new BigDecimal[] { new BigDecimal(45821.02) };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public BigDecimal getDecimal() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return new BigDecimal(46851.2);
+ } else {
+ return new BigDecimal(45821.02);
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testForLessThanEqualToExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
@@ -319,6 +430,38 @@ public class LessThanEqualToExpressionUnitTest {
}
+ @Test public void testEvaluateForLessThanEqualToExpressionWithIsNullReturnTrue1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
+ right.setColIndex(0);
+ lessThanEqualToExpression = new LessThanEqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15856 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public boolean isNull() {
+ if (isFirst) {
+ isFirst = false;
+ return false;
+ }
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15856;
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
@Test public void testEvaluateForLessThanEqualToExpressionWithLeftAndRightDifferentDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("name", DataTypes.STRING);
@@ -351,6 +494,38 @@ public class LessThanEqualToExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForLessThanEqualToExpressionWithLeftAndRightDifferentDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataTypes.INT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataTypes.STRING);
+ right.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "S" };
+ Integer[] row1 = { 1450 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 84;
+ } else {
+ return 1450;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
@Test public void testForLessThanEqualToExpressionWithGetString() throws Exception {
ColumnExpression right = new ColumnExpression("right_name", DataTypes.STRING);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
index e038a56..395750a 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
@@ -72,6 +72,37 @@ public class LessThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForLessThanExpressionWithStringDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataTypes.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataTypes.STRING);
+ right.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "First String Value" };
+ String[] row1 = { "string1" };
+ Object objectRow[] = { row, row1 };
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public String getString() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return "string1";
+
+ } else {
+ return "First String Value";
+
+ }
+
+ }
+ };
+ value.setValues(objectRow);
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testEvaluateForLessThanExpressionWithShortDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
@@ -106,6 +137,40 @@ public class LessThanExpressionUnitTest {
}
+ @Test public void testEvaluateForLessThanExpressionWithShortDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("id", DataTypes.SHORT);
+ left.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 7052 };
+ Short[] row1 = { 7450 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Short getShort() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 7450;
+
+ } else {
+ return 7052;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
@Test public void testEvaluateForLessThanExpressionWithDoubleDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DOUBLE);
@@ -217,6 +282,51 @@ public class LessThanExpressionUnitTest {
}
}
+ @Test public void testEvaluateForLessThanExpressionWithTimestampDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("timestamp", DataTypes.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("timestamp", DataTypes.TIMESTAMP);
+ right.setColIndex(1);
+
+ lessThanExpression = new LessThanExpression(left, right);
+
+ RowImpl value = new RowImpl();
+
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+
+ Date date1 = dateFormat.parse("24/09/2007");
+ long time1 = date1.getTime();
+ Timestamp[] row1 = { new Timestamp(time1) };
+
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getTime() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1190592000L;
+ } else {
+ return 1190505600L;
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
@Test public void testEvaluateForLessThanExpressionWithLongDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.LONG);
@@ -247,6 +357,36 @@ public class LessThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForLessThanExpressionWithLongDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.LONG);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataTypes.LONG);
+ left.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 14523656L };
+ Long[] row1 = { 12456325L };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getLong() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 14523656L;
+ } else {
+ return 12456325L;
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testEvaluateForLessThanExpressionWithDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
@@ -316,6 +456,38 @@ public class LessThanExpressionUnitTest {
}
+ @Test public void testEvaluateForLessThanExpressionWithIsNullReturnTrue1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
+ right.setColIndex(0);
+ lessThanExpression = new LessThanExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public boolean isNull() {
+ if (isFirst) {
+ isFirst = false;
+ return false;
+ }
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
@Test public void testEvaluateForLessThanExpressionWithLeftAndRightDifferentDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("name", DataTypes.STRING);
@@ -348,6 +520,38 @@ public class LessThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForLessThanExpressionWithLeftAndRightDifferentDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataTypes.INT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataTypes.STRING);
+ right.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "S" };
+ Integer[] row1 = { 1864 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 84;
+ } else {
+ return 1864;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
@Test public void testForLessThanExpressionWithGetString() throws Exception {
ColumnExpression right = new ColumnExpression("right_name", DataTypes.STRING);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
index b885fc8..17e5eb7 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
@@ -33,6 +33,7 @@ import java.util.ArrayList;
import static org.hamcrest.core.Is.is;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
public class ListExpressionUnitTest {
@@ -58,4 +59,25 @@ public class ListExpressionUnitTest {
ExpressionResult result = listExpression.evaluate(value);
assertThat(expected_value, is(equalTo(result.getList().get(0).getString())));
}
+
+ @Test public void testGetString() throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataTypes.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataTypes.STRING);
+ right.setColIndex(1);
+
+ List<Expression> children = new ArrayList<>();
+ children.add(left);
+ children.add(right);
+
+ listExpression = new ListExpression(children);
+ RowImpl value = new RowImpl();
+ String row = "Row is for left";
+ String row1 = "I am row 1";
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+ String expected_value = "ListExpression(ColumnExpression(left_name);ColumnExpression(right_name);)";
+ String exresult = listExpression.getString();
+ assertTrue(exresult.equals(expected_value));
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
index 34448f4..d0ee21e 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
@@ -108,7 +108,34 @@ public class NotEqualsExpressionUnitTest {
}
- @Test public void testEvaluateForNotEqualsExpressionWithIntDataType()
+ @Test public void testEvaluateForNotEqualsExpressionWithShortDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+
+ ColumnExpression left = new ColumnExpression("left_id", DataTypes.SHORT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_id", DataTypes.SHORT);
+ right.setColIndex(1);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Short[] row1 = { 16 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+
+ @Mock public Short getShort() {
+ return 15;
+
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithIntDataType1()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("right_number", DataTypes.INT);
right.setColIndex(1);
@@ -139,6 +166,29 @@ public class NotEqualsExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForNotEqualsExpressionWithIntDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_number", DataTypes.INT);
+ right.setColIndex(1);
+ ColumnExpression left = new ColumnExpression("left_number", DataTypes.INT);
+ left.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ Integer[] row = { 15 };
+ Integer[] row1 = { 16 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testEvaluateForNotEqualsExpressionWithDoubleDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DOUBLE);
@@ -200,6 +250,31 @@ public class NotEqualsExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForNotEqualsExpressionWithLongDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataTypes.LONG);
+ right.setColIndex(1);
+ ColumnExpression left = new ColumnExpression("left_contact", DataTypes.LONG);
+ left.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 1234567654321L };
+ Long[] row1 = { 12345676541L };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getLong() {
+ return 1234567654321L;
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testEvaluateForNotEqualsExpressionWithTimestampDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
try {
@@ -245,6 +320,46 @@ public class NotEqualsExpressionUnitTest {
}
}
+ @Test public void testEvaluateForNotEqualsExpressionWithTimestampDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("left_timestamp", DataTypes.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_timestamp", DataTypes.TIMESTAMP);
+ right.setColIndex(1);
+
+ notEqualsExpression = new NotEqualsExpression(left, right);
+
+ RowImpl value = new RowImpl();
+
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+
+ Date date1 = dateFormat.parse("24/09/2007");
+ long time1 = date1.getTime();
+ Timestamp[] row1 = { new Timestamp(time1) };
+
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getTime() {
+ return 1190592000L;
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
@Test public void testForNotEqualsExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
@@ -287,6 +402,31 @@ public class NotEqualsExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateWithForNotEqualsExpressionDecimalDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
+ right.setColIndex(1);
+ ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
+ left.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ BigDecimal[] row = new BigDecimal[] { new BigDecimal(12345.0) };
+ BigDecimal[] row1 = new BigDecimal[] { new BigDecimal(1235445.0) };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public BigDecimal getDecimal() {
+ return new BigDecimal(12345.0);
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testEvaluateForNotEqualsExpressionWithIsNullReturnTrue()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
@@ -314,6 +454,38 @@ public class NotEqualsExpressionUnitTest {
}
+ @Test public void testEvaluateForNotEqualsExpressionWithIsNullReturnTrue1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataTypes.SHORT);
+ right.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 150 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ boolean isFirst = true;
+ @Mock public boolean isNull() {
+ if (isFirst) {
+ isFirst = false;
+ return false;
+ }
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 150;
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
@Test public void testEvaluateForNotEqualsExpressionWithLeftAndRightDifferentDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("name", DataTypes.STRING);
@@ -344,6 +516,36 @@ public class NotEqualsExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForNotEqualsExpressionWithLeftAndRightDifferentDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataTypes.INT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataTypes.STRING);
+ right.setColIndex(1);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row1 = { "S" };
+ Integer[] row = { 14 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 84;
+ } else {
+ return 14;
+ }
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
@Test public void testForNotEqualsExpressionWithGetString() throws Exception {
ColumnExpression right = new ColumnExpression("right_name", DataTypes.STRING);
ColumnExpression left = new ColumnExpression("left_name", DataTypes.STRING);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
index d31361b..54791cc 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
@@ -36,6 +36,7 @@ import mockit.MockUp;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class NotInExpressionUnitTest {
@@ -143,6 +144,35 @@ public class NotInExpressionUnitTest {
assertTrue(result.getBoolean());
}
+ @Test public void testEvaluateForNotInExpressionWithDoubleDataType1()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_contact", DataTypes.DOUBLE);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DOUBLE);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ RowImpl value = new RowImpl();
+ Double row = 44521D;
+ Double row1 = 44521.023D;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Double getDouble() {
+ return 44521.023D;
+ }
+ };
+
+ ExpressionResult result = notInExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ }
+
@Test public void testEvaluateForNotInExpressionWithLongDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("left_contact", DataTypes.LONG);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/AndExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/AndExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/AndExpressionTest.java
index 6bb6900..5cc6a7b 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/AndExpressionTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/AndExpressionTest.java
@@ -71,6 +71,74 @@ public class AndExpressionTest {
assertTrue(actualValue instanceof ExpressionResult);
}
+ @Test public void testEvaluate1() throws FilterIllegalMemberException,
+ FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResult = new ExpressionResult(DataTypes.BOOLEAN, "true");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ return expressionResult;
+ }
+ };
+
+ assertTrue(andExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate2() throws FilterIllegalMemberException,
+ FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResultRight = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ final ExpressionResult expressionResultLeft = new ExpressionResult(DataTypes.BOOLEAN, "true");
+ new MockUp<ColumnExpression>() {
+ boolean isLeft = true;
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ if (isLeft) {
+ isLeft = false;
+ return expressionResultLeft;
+ }
+ return expressionResultRight;
+ }
+ };
+
+ assertTrue(andExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate3() throws FilterIllegalMemberException,
+ FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResultRight = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ final ExpressionResult expressionResultLeft = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ new MockUp<ColumnExpression>() {
+ boolean isLeft = true;
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ if (isLeft) {
+ isLeft = false;
+ return expressionResultLeft;
+ }
+ return expressionResultRight;
+ }
+ };
+
+ assertTrue(andExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate4()
+ throws FilterIllegalMemberException, FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResult = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ return expressionResult;
+ }
+ };
+
+ assertTrue(andExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
@Test public void testGetString() {
String actualValue = andExpression.getString();
String expectedValue = "And(ColumnExpression(IMEI),ColumnExpression(IMEI))";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/FalseExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/FalseExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/FalseExpressionTest.java
index 59bf0b5..85486e2 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/FalseExpressionTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/FalseExpressionTest.java
@@ -22,6 +22,7 @@ import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.junit.Before;
@@ -43,4 +44,15 @@ public class FalseExpressionTest {
ExpressionResult actualValue = falseExpression.evaluate(rowImpl);
assertEquals(new ExpressionResult(DataTypes.BOOLEAN, false), actualValue);
}
+
+ @Test public void testGetString() {
+ String actualValue = falseExpression.getString();
+ String expectedValue = "False(ColumnExpression(IMEI)";
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testFilterExpressionType() {
+ ExpressionType actualValue = falseExpression.getFilterExpressionType();
+ assertEquals(ExpressionType.FALSE, actualValue);
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/OrExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/OrExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/OrExpressionTest.java
index 6acfaea..83c3269 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/OrExpressionTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/OrExpressionTest.java
@@ -61,6 +61,74 @@ public class OrExpressionTest {
assertTrue(orExpression.evaluate(rowImpl) instanceof ExpressionResult);
}
+ @Test public void testEvaluate1() throws FilterIllegalMemberException,
+ FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResult = new ExpressionResult(DataTypes.BOOLEAN, "true");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ return expressionResult;
+ }
+ };
+
+ assertTrue(orExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate2() throws FilterIllegalMemberException,
+ FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResultRight = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ final ExpressionResult expressionResultLeft = new ExpressionResult(DataTypes.BOOLEAN, "true");
+ new MockUp<ColumnExpression>() {
+ boolean isLeft = true;
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ if (isLeft) {
+ isLeft = false;
+ return expressionResultLeft;
+ }
+ return expressionResultRight;
+ }
+ };
+
+ assertTrue(orExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate3() throws FilterIllegalMemberException,
+ FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResultRight = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ final ExpressionResult expressionResultLeft = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ new MockUp<ColumnExpression>() {
+ boolean isLeft = true;
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ if (isLeft) {
+ isLeft = false;
+ return expressionResultLeft;
+ }
+ return expressionResultRight;
+ }
+ };
+
+ assertTrue(orExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate4()
+ throws FilterIllegalMemberException, FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResult = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ return expressionResult;
+ }
+ };
+
+ assertTrue(orExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
@Test(expected = Exception.class) public void testEvaluateForDefault()
throws FilterUnsupportedException, FilterIllegalMemberException {
RowImpl rowImpl = new RowImpl();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/RangeExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/RangeExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/RangeExpressionTest.java
new file mode 100644
index 0000000..b76a3db
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/RangeExpressionTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import mockit.Mock;
+import mockit.MockUp;
+
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+public class RangeExpressionTest {
+ private RangeExpression rangeExpression;
+
+ @Before public void setUp() {
+ ColumnExpression leftExpression = new ColumnExpression("IMEI", DataTypes.BOOLEAN);
+ ColumnExpression rightExpression = new ColumnExpression("IMEI", DataTypes.BOOLEAN);
+ rangeExpression = new RangeExpression(leftExpression, rightExpression);
+ }
+
+ @Test public void testGetString() {
+ String actualValue = rangeExpression.getString();
+ String expectedValue = "Range(ColumnExpression(IMEI),ColumnExpression(IMEI))";
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testEvaluate() throws FilterIllegalMemberException, FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResult = new ExpressionResult(DataTypes.BOOLEAN, "test");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ return expressionResult;
+ }
+ };
+
+ assertTrue(rangeExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate1()
+ throws FilterIllegalMemberException, FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResult = new ExpressionResult(DataTypes.BOOLEAN, "true");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ return expressionResult;
+ }
+ };
+
+ assertTrue(rangeExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate2()
+ throws FilterIllegalMemberException, FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResultRight = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ final ExpressionResult expressionResultLeft = new ExpressionResult(DataTypes.BOOLEAN, "true");
+ new MockUp<ColumnExpression>() {
+ boolean isLeft = true;
+
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ if (isLeft) {
+ isLeft = false;
+ return expressionResultLeft;
+ }
+ return expressionResultRight;
+ }
+ };
+
+ assertTrue(rangeExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate3()
+ throws FilterIllegalMemberException, FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResultRight = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ final ExpressionResult expressionResultLeft = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ new MockUp<ColumnExpression>() {
+ boolean isLeft = true;
+
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ if (isLeft) {
+ isLeft = false;
+ return expressionResultLeft;
+ }
+ return expressionResultRight;
+ }
+ };
+
+ assertTrue(rangeExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test public void testEvaluate4()
+ throws FilterIllegalMemberException, FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResult = new ExpressionResult(DataTypes.BOOLEAN, "false");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ return expressionResult;
+ }
+ };
+
+ assertTrue(rangeExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test(expected = Exception.class) public void testEvaluateForDefault()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { true });
+ final ExpressionResult expressionResult = new ExpressionResult(DataTypes.STRING, "test");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ return expressionResult;
+ }
+ };
+ rangeExpression.evaluate(rowImpl);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/TrueExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/TrueExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/TrueExpressionTest.java
new file mode 100644
index 0000000..77afc85
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/TrueExpressionTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TrueExpressionTest {
+ private TrueExpression trueExpression;
+
+ @Before public void setUp() {
+ ColumnExpression columnExpression = new ColumnExpression("IMEI", DataTypes.BOOLEAN);
+ trueExpression = new TrueExpression(columnExpression);
+ }
+
+ @Test public void testEvaluate() throws FilterUnsupportedException, FilterIllegalMemberException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { true });
+ ExpressionResult actualValue = trueExpression.evaluate(rowImpl);
+ assertEquals(new ExpressionResult(DataTypes.BOOLEAN, true), actualValue);
+ }
+
+ @Test public void testGetString() {
+ String actualValue = trueExpression.getString();
+ String expectedValue = "True(ColumnExpression(IMEI)";
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testFilterExpressionType() {
+ ExpressionType actualValue = trueExpression.getFilterExpressionType();
+ assertEquals(ExpressionType.TRUE, actualValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
index f294454..74f9625 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessorTest.java
@@ -36,7 +36,6 @@ public class FilterExpressionProcessorTest extends AbstractDictionaryCacheTest {
new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
this.carbonStorePath = props.getProperty("storePath", "carbonStore");
columnSchema = new ColumnSchema();
- columnSchema.setColumnar(true);
columnSchema.setColumnName("IMEI");
columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema.setDataType(DataTypes.STRING);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
index 9c7c26c..cbc0203 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -66,7 +66,6 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
this.carbonStorePath = props.getProperty("storePath", "carbonStore");
columnSchema = new ColumnSchema();
- columnSchema.setColumnar(true);
columnSchema.setColumnName("IMEI");
columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema.setDataType(DataTypes.STRING);
@@ -232,17 +231,14 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
new MultiDimKeyVarLengthGenerator(keys);
int ordinal = 1;
int keyOrdinal = 1;
- int columnGroupOrdinal = 1;
int complexTypeOrdinal = 1;
ColumnSchema columnSchema = new ColumnSchema();
- columnSchema.setColumnar(true);
columnSchema.setColumnName("IMEI");
columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema.setDataType(DataTypes.STRING);
columnSchema.setDimensionColumn(true);
CarbonDimension carbonDimension =
- new CarbonDimension(columnSchema, ordinal, keyOrdinal, columnGroupOrdinal,
- complexTypeOrdinal);
+ new CarbonDimension(columnSchema, ordinal, keyOrdinal, complexTypeOrdinal);
byte[] expectedResult = new byte[] { 1 };
byte[] actualResult =
FilterUtil.getMaskKey(surrogate, carbonDimension, multiDimKeyVarLengthGenerator);
@@ -257,7 +253,6 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
boolean isIncludeFilter = true;
int ordinal = 1;
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
@@ -288,7 +283,6 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
boolean isIncludeFilter = true;
int ordinal = 1;
ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataTypes.STRING);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
index 92c2e85..b9e90d6 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
@@ -194,7 +194,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
oldTime = oldTime + System.currentTimeMillis() - start;
start = System.currentTimeMillis();
- BitSet bitNew = this.setFilterdIndexToBitSetNew((FixedLengthDimensionColumnPage) dimensionColumnDataChunk, dataChunkSize,
+ BitSet bitNew = this.setFilterdIndexToBitSetNew(dimensionColumnDataChunk, dataChunkSize,
filterKeys);
newTime = newTime + System.currentTimeMillis() - start;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapperTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapperTest.java b/core/src/test/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapperTest.java
index e5104fa..78dca73 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapperTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapperTest.java
@@ -23,6 +23,7 @@ import org.junit.Test;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertNotSame;
import static junit.framework.TestCase.assertTrue;
public class ByteArrayWrapperTest {
@@ -30,6 +31,9 @@ public class ByteArrayWrapperTest {
byte[] dictionaryKey = new byte[] { 1 };
byte[][] complexTypesKeys = { { 1 }, { 1 } };
byte[][] noDictionaryKeys = new byte[][] { { 1 }, { 1 } };
+ byte[] dictionaryKey1 = new byte[] { 2 };
+ byte[][] complexTypesKeys1 = { { 2 }, { 1 } };
+ byte[][] noDictionaryKeys1 = new byte[][] { { 2 }, { 1 } };
@BeforeClass
@@ -91,6 +95,33 @@ public class ByteArrayWrapperTest {
assertTrue(result);
}
+ @Test
+
+ public void testEqualsForFirstElementComplexTypesKeysAndOther1() {
+ ByteArrayWrapper other = new ByteArrayWrapper();
+ other.setComplexTypesKeys(complexTypesKeys);
+ other.setDictionaryKey(dictionaryKey);
+ other.setNoDictionaryKeys(noDictionaryKeys);
+ byteArrayWrapper.setComplexTypesKeys(complexTypesKeys);
+ byteArrayWrapper.setDictionaryKey(dictionaryKey);
+ byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys1);
+ boolean result = byteArrayWrapper.equals(other);
+ assertFalse(result);
+ }
+
+ @Test
+ public void testEqualsForFirstElementComplexTypesKeysAndOther2() {
+ ByteArrayWrapper other = new ByteArrayWrapper();
+ other.setComplexTypesKeys(complexTypesKeys);
+ other.setDictionaryKey(dictionaryKey);
+ other.setNoDictionaryKeys(noDictionaryKeys);
+ byteArrayWrapper.setComplexTypesKeys(complexTypesKeys1);
+ byteArrayWrapper.setDictionaryKey(dictionaryKey);
+ byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
+ boolean result = byteArrayWrapper.equals(other);
+ assertFalse(result);
+ }
+
@Test public void testCompareTo() {
byteArrayWrapper.setDictionaryKey(dictionaryKey);
ByteArrayWrapper other = new ByteArrayWrapper();
@@ -103,4 +134,30 @@ public class ByteArrayWrapperTest {
int expectedResult = 0;
assertEquals(expectedResult, actualResult);
}
+
+ @Test public void testCompareTo1() {
+ byteArrayWrapper.setDictionaryKey(dictionaryKey1);
+ ByteArrayWrapper other = new ByteArrayWrapper();
+ other.setDictionaryKey(dictionaryKey1);
+ other.setNoDictionaryKeys(noDictionaryKeys);
+ other.setComplexTypesKeys(complexTypesKeys);
+ byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
+ byteArrayWrapper.setComplexTypesKeys(complexTypesKeys1);
+ int actualResult = byteArrayWrapper.compareTo(other);
+ int expectedResult = 0;
+ assertNotSame(expectedResult, actualResult);
+ }
+
+ @Test public void testCompareTo2() {
+ byteArrayWrapper.setDictionaryKey(dictionaryKey);
+ ByteArrayWrapper other = new ByteArrayWrapper();
+ other.setDictionaryKey(dictionaryKey);
+ other.setNoDictionaryKeys(noDictionaryKeys);
+ other.setComplexTypesKeys(complexTypesKeys);
+ byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys1);
+ byteArrayWrapper.setComplexTypesKeys(complexTypesKeys1);
+ int actualResult = byteArrayWrapper.compareTo(other);
+ int expectedResult = 0;
+ assertNotSame(expectedResult, actualResult);
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/util/CarbonMergerUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMergerUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMergerUtilTest.java
deleted file mode 100644
index 53d425f..0000000
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMergerUtilTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.util;
-
-import static junit.framework.TestCase.*;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.Test;
-
-import static org.apache.carbondata.core.util.CarbonMergerUtil.getCardinalityFromLevelMetadata;
-
-public class CarbonMergerUtilTest {
-
- @Test public void testGetCardinalityFromLevelMetadata() throws Exception {
- final int[] localCardinality = { 1, 2, 3, 4, 5, 6 };
- new MockUp<CarbonUtil>() {
- @SuppressWarnings("unused") @Mock
- public int[] getCardinalityFromLevelMetadataFile(String levelPath) {
- return localCardinality;
- }
- };
- int[] result = getCardinalityFromLevelMetadata("STORE_PATH", "table1");
- assertEquals(result, localCardinality);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index d75f323..5520bfb 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -32,7 +32,6 @@ import java.util.Map;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
-import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
@@ -104,16 +103,6 @@ public class CarbonUtilTest {
}
}
- @Test public void testToGetColGroupModel() {
- int[][] cardinality = { { 10, 20, 30 }, { 20, 30 }, {} };
- ColumnGroupModel actualResult = CarbonUtil.getColGroupModel(cardinality);
- assertEquals(actualResult.getNoOfColumnStore(), 3);
- int[] expectedResult = { 3, 2, 0 };
- for (int i = 0; i < actualResult.getColumnSplit().length; i++) {
- assertEquals(actualResult.getColumnSplit()[i], expectedResult[i]);
- }
- }
-
@Test public void testToGetIncrementedCardinalityFullyFilled() {
int[] cardinality = { 200, 20, 0, 10 };
int[] actualResult = CarbonUtil.getIncrementedCardinalityFullyFilled(cardinality);
@@ -298,17 +287,6 @@ public class CarbonUtilTest {
assertEquals(result, 5);
}
-
- @Test public void testToGetCardinalityFromLevelMetadataFileForInvalidPath()
- throws IOException, InterruptedException {
- try {
- int[] cardinality = CarbonUtil.getCardinalityFromLevelMetadataFile("");
- assertTrue(false);
- } catch (Exception e) {
- assertTrue(true);
- }
- }
-
@Test public void testToUnescapeChar() {
String[] input = { "\\001", "\\t", "\\r", "\\b", "\\n", "\\f" };
String[] output = { "\001", "\t", "\r", "\b", "\n", "\f" };
@@ -522,14 +500,14 @@ public class CarbonUtilTest {
encoding.add(Encoding.DICTIONARY);
column1Schema.setEncodingList(encoding);
ProjectionDimension
- column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1));
column2Schema.setColumnName("Column2");
List<Encoding> encoding2 = new ArrayList<>();
encoding2.add(Encoding.DELTA);
column2Schema.setEncodingList(encoding2);
ProjectionDimension
- column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1));
ProjectionDimension[] queryDimensions = { column1, column2 };
@@ -548,14 +526,14 @@ public class CarbonUtilTest {
encoding.add(Encoding.DIRECT_DICTIONARY);
column1Schema.setEncodingList(encoding);
ProjectionDimension
- column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1));
column2Schema.setColumnName("Column2");
List<Encoding> encoding2 = new ArrayList<>();
encoding2.add(Encoding.DELTA);
column2Schema.setEncodingList(encoding2);
ProjectionDimension
- column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1));
ProjectionDimension[] queryDimensions = { column1, column2 };
@@ -572,12 +550,12 @@ public class CarbonUtilTest {
column1Schema.setColumnName("Column1");
column1Schema.setDataType(DataTypes.DATE);
ProjectionDimension
- column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1));
column2Schema.setColumnName("Column2");
column2Schema.setDataType(DataTypes.createDefaultArrayType());
ProjectionDimension
- column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1));
ProjectionDimension[] queryDimensions = { column1, column2 };
@@ -617,10 +595,10 @@ public class CarbonUtilTest {
column1Schema.setColumnName("Column1");
column2Schema.setColumnName("Column2");
List<CarbonDimension> carbonDimension = new ArrayList<>();
- carbonDimension.add(new CarbonDimension(column1Schema, 1, 1, 1, 1));
- carbonDimension.add(new CarbonDimension(column2Schema, 2, 1, 2, 1));
+ carbonDimension.add(new CarbonDimension(column1Schema, 1, 1, 1));
+ carbonDimension.add(new CarbonDimension(column2Schema, 2, 1, 1));
assertEquals(CarbonUtil.findDimension(carbonDimension, "Column1"),
- new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ new CarbonDimension(column1Schema, 1, 1, 1));
}
@Test public void testToGetFormattedCardinality() {
@@ -649,8 +627,8 @@ public class CarbonUtilTest {
column1Schema.setColumnName("Column1");
column2Schema.setColumnName("Column2");
List<CarbonDimension> carbonDimension = new ArrayList<>();
- carbonDimension.add(new CarbonDimension(column1Schema, 1, 1, 1, 1));
- carbonDimension.add(new CarbonDimension(column2Schema, 2, 2, 2, 1));
+ carbonDimension.add(new CarbonDimension(column1Schema, 1, 1, 1));
+ carbonDimension.add(new CarbonDimension(column2Schema, 2, 2, 1));
List<CarbonMeasure> carbonMeasure = new ArrayList<>();
carbonMeasure.add(new CarbonMeasure(column1Schema, 1));
@@ -776,17 +754,14 @@ public class CarbonUtilTest {
ColumnSchema column2Schema = new ColumnSchema();
ColumnSchema column3Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
- column1Schema.setColumnar(true);
column1Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.DICTIONARY));
column2Schema.setColumnName("Column2");
- column2Schema.setColumnar(false);
column2Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.DICTIONARY));
column3Schema.setColumnName("Column3");
- column3Schema.setColumnar(true);
column3Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.INVERTED_INDEX));
- CarbonDimension carbonDimension = new CarbonDimension(column1Schema, 1, 1, 1, 1);
- CarbonDimension carbonDimension2 = new CarbonDimension(column2Schema, 2, 2, 2, 2);
- CarbonDimension carbonDimension3 = new CarbonDimension(column3Schema, 3, 3, 3, 3);
+ CarbonDimension carbonDimension = new CarbonDimension(column1Schema, 1, 1, 1);
+ CarbonDimension carbonDimension2 = new CarbonDimension(column2Schema, 2, 2, 2);
+ CarbonDimension carbonDimension3 = new CarbonDimension(column3Schema, 3, 3, 3);
List<CarbonDimension> carbonDimensions =
Arrays.asList(carbonDimension, carbonDimension2, carbonDimension3);
boolean[] result = CarbonUtil.identifyDimensionType(carbonDimensions);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
index 4fb5dcc..9b8be79 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
@@ -82,7 +82,7 @@ public class RangeFilterProcessorTest {
empColumnSchema.setDimensionColumn(true);
empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
empColumnSchema.setDataType(DataTypes.STRING);
- CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+ CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0);
cola.setDimension(empDimension);
Expression greaterThan =
@@ -123,7 +123,7 @@ public class RangeFilterProcessorTest {
empColumnSchema.setDimensionColumn(true);
empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
empColumnSchema.setDataType(DataTypes.STRING);
- CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+ CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0);
cola.setDimension(empDimension);
Expression greaterThan =
@@ -162,7 +162,7 @@ public class RangeFilterProcessorTest {
empColumnSchema.setDimensionColumn(true);
empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
empColumnSchema.setDataType(DataTypes.STRING);
- CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+ CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0);
ColumnExpression cola1 = new ColumnExpression("a", DataTypes.STRING);
cola1.setDimension(true);
@@ -237,7 +237,7 @@ public class RangeFilterProcessorTest {
empColumnSchema.setDimensionColumn(true);
empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
empColumnSchema.setDataType(DataTypes.STRING);
- CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+ CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0);
ColumnExpression cola1 = new ColumnExpression("a", DataTypes.STRING);
cola1.setDimension(true);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
index d5500e1..9f2dc8e 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
@@ -119,9 +119,9 @@ public class CarbonDictionaryWriterImplTest {
* prepare the dataset required for running test cases
*/
private void prepareDataSet() {
- dataSet1 = Arrays.asList(new String[] { "a", "b" });
- dataSet2 = Arrays.asList(new String[] { "c", "d" });
- dataSet3 = Arrays.asList(new String[] { "e", "f" });
+ dataSet1 = Arrays.asList("a", "b");
+ dataSet2 = Arrays.asList("c", "d");
+ dataSet3 = Arrays.asList("e", "f");
}
/**
[7/8] carbondata git commit: [CARBONDATA-2720] Remove dead code
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
deleted file mode 100644
index 4fe6d1b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.block;
-
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.carbondata.core.cache.Cacheable;
-import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
-
-/**
- * SegmentTaskIndexWrapper class holds the taskIdToTableSegmentMap
- */
-public class SegmentTaskIndexWrapper implements Cacheable {
-
- /**
- * task_id to table segment index map
- */
- private Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> taskIdToTableSegmentMap;
- /**
- * atomic integer to maintain the access count for a column access
- */
- protected AtomicInteger accessCount = new AtomicInteger();
-
- /**
- * Table block meta size.
- */
- protected AtomicLong memorySize = new AtomicLong();
-
- private Long refreshedTimeStamp;
- public SegmentTaskIndexWrapper(
- Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> taskIdToTableSegmentMap) {
- this.taskIdToTableSegmentMap = taskIdToTableSegmentMap;
- }
-
- public Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> getTaskIdToTableSegmentMap() {
- return taskIdToTableSegmentMap;
- }
-
- public void setTaskIdToTableSegmentMap(
- Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> taskIdToTableSegmentMap) {
- this.taskIdToTableSegmentMap = taskIdToTableSegmentMap;
- }
-
- /**
- * return segment size
- *
- * @param memorySize
- */
- public void setMemorySize(long memorySize) {
- this.memorySize.set(memorySize);
- }
-
- /**
- * returns the timestamp
- *
- * @return
- */
- @Override public long getFileTimeStamp() {
- return 0;
- }
-
- /**
- * returns the access count
- *
- * @return
- */
- @Override public int getAccessCount() {
- return accessCount.get();
- }
-
- /**
- * returns the memory size
- *
- * @return
- */
- @Override public long getMemorySize() {
- return memorySize.get();
- }
-
- /**
- * The method is used to set the access count
- */
- public void incrementAccessCount() {
- accessCount.incrementAndGet();
- }
-
- /**
- * This method will release the objects and set default value for primitive types
- */
- public void clear() {
- decrementAccessCount();
- }
-
- /**
- * This method will decrement the access count for a column by 1
- * whenever a column usage is complete
- */
- private void decrementAccessCount() {
- if (accessCount.get() > 0) {
- accessCount.decrementAndGet();
- }
- }
-
- public Long getRefreshedTimeStamp() {
- return refreshedTimeStamp;
- }
-
- public void setRefreshedTimeStamp(Long refreshedTimeStamp) {
- this.refreshedTimeStamp = refreshedTimeStamp;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
index 15840bc..50fa09a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
@@ -16,7 +16,6 @@
*/
package org.apache.carbondata.core.datastore.chunk;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
/**
@@ -31,7 +30,7 @@ public interface DimensionColumnPage {
* @param data data to filed
* @return how many bytes was copied
*/
- int fillRawData(int rowId, int offset, byte[] data, KeyStructureInfo restructuringInfo);
+ int fillRawData(int rowId, int offset, byte[] data);
/**
* It uses to convert column data to dictionary integer value
@@ -39,31 +38,26 @@ public interface DimensionColumnPage {
* @param rowId
* @param chunkIndex
* @param outputSurrogateKey
- * @param restructuringInfo @return
+ * @return
*/
- int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
- KeyStructureInfo restructuringInfo);
+ int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey);
/**
* Fill the data to vector
* @param vectorInfo
* @param chunkIndex
- * @param restructuringInfo
* @return next column index
*/
- int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
- KeyStructureInfo restructuringInfo);
+ int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex);
/**
* Fill the data to vector
* @param filteredRowId
* @param vectorInfo
* @param chunkIndex
- * @param restructuringInfo
* @return next column index
*/
- int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int chunkIndex,
- KeyStructureInfo restructuringInfo);
+ int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int chunkIndex);
/**
* Below method to get the data based in row id
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
index e25fcb9..c815e4d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
@@ -21,7 +21,6 @@ import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFacto
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
@@ -57,11 +56,9 @@ public class FixedLengthDimensionColumnPage extends AbstractDimensionColumnPage
* @param rowId row id of the chunk
* @param offset offset from which data need to be filed
* @param data data to filed
- * @param keyStructureInfo define the structure of the key
* @return how many bytes was copied
*/
- @Override public int fillRawData(int rowId, int offset, byte[] data,
- KeyStructureInfo keyStructureInfo) {
+ @Override public int fillRawData(int rowId, int offset, byte[] data) {
dataChunkStore.fillRow(rowId, data, offset);
return dataChunkStore.getColumnValueSize();
}
@@ -72,11 +69,9 @@ public class FixedLengthDimensionColumnPage extends AbstractDimensionColumnPage
* @param rowId
* @param chunkIndex
* @param outputSurrogateKey
- * @param restructuringInfo
* @return
*/
- @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
- KeyStructureInfo restructuringInfo) {
+ @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey) {
outputSurrogateKey[chunkIndex] = dataChunkStore.getSurrogate(rowId);
return chunkIndex + 1;
}
@@ -86,11 +81,9 @@ public class FixedLengthDimensionColumnPage extends AbstractDimensionColumnPage
*
* @param vectorInfo
* @param chunkIndex
- * @param restructuringInfo
* @return next column index
*/
- @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
- KeyStructureInfo restructuringInfo) {
+ @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex) {
ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
int offset = columnVectorInfo.offset;
int vectorOffset = columnVectorInfo.vectorOffset;
@@ -127,11 +120,10 @@ public class FixedLengthDimensionColumnPage extends AbstractDimensionColumnPage
* @param filteredRowId
* @param vectorInfo
* @param chunkIndex
- * @param restructuringInfo
* @return next column index
*/
@Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo,
- int chunkIndex, KeyStructureInfo restructuringInfo) {
+ int chunkIndex) {
ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
int offset = columnVectorInfo.offset;
int vectorOffset = columnVectorInfo.vectorOffset;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
index aea594c..a404ff7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
@@ -65,11 +64,9 @@ public class VariableLengthDimensionColumnPage extends AbstractDimensionColumnPa
* @param rowId row id of the chunk
* @param offset offset from which data need to be filed
* @param data data to filed
- * @param restructuringInfo define the structure of the key
* @return how many bytes was copied
*/
- @Override public int fillRawData(int rowId, int offset, byte[] data,
- KeyStructureInfo restructuringInfo) {
+ @Override public int fillRawData(int rowId, int offset, byte[] data) {
// no required in this case because this column chunk is not the part if
// mdkey
return 0;
@@ -81,11 +78,9 @@ public class VariableLengthDimensionColumnPage extends AbstractDimensionColumnPa
* @param rowId
* @param chunkIndex
* @param outputSurrogateKey
- * @param restructuringInfo
* @return
*/
- @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
- KeyStructureInfo restructuringInfo) {
+ @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey) {
return chunkIndex + 1;
}
@@ -101,11 +96,9 @@ public class VariableLengthDimensionColumnPage extends AbstractDimensionColumnPa
*
* @param vectorInfo
* @param chunkIndex
- * @param restructuringInfo
* @return next column index
*/
- @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
- KeyStructureInfo restructuringInfo) {
+ @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex) {
ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
CarbonColumnVector vector = columnVectorInfo.vector;
int offset = columnVectorInfo.offset;
@@ -125,11 +118,10 @@ public class VariableLengthDimensionColumnPage extends AbstractDimensionColumnPa
* @param filteredRowId
* @param vectorInfo
* @param chunkIndex
- * @param restructuringInfo
* @return next column index
*/
@Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo,
- int chunkIndex, KeyStructureInfo restructuringInfo) {
+ int chunkIndex) {
ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
CarbonColumnVector vector = columnVectorInfo.vector;
int offset = columnVectorInfo.offset;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
index f4712ba..f95ff11 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
@@ -24,7 +24,6 @@ import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
import org.apache.carbondata.core.util.ByteUtil;
@@ -48,25 +47,22 @@ public class ColumnPageWrapper implements DimensionColumnPage {
}
@Override
- public int fillRawData(int rowId, int offset, byte[] data, KeyStructureInfo restructuringInfo) {
+ public int fillRawData(int rowId, int offset, byte[] data) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
- KeyStructureInfo restructuringInfo) {
+ public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
- KeyStructureInfo restructuringInfo) {
+ public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int chunkIndex,
- KeyStructureInfo restructuringInfo) {
+ public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int chunkIndex) {
throw new UnsupportedOperationException("internal error");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
deleted file mode 100644
index e2a4161..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.columnar;
-
-public class ColumnGroupModel {
-
- /**
- * number of columns in columnar block
- */
- private int[] columnSplit;
-
- /**
- * total number of columns
- */
- private int noOfColumnsStore;
-
- /**
- * column groups
- * e.g
- * {{0,1,2},3,4,{5,6}}
- */
- private int[][] columnGroups;
-
- /**
- * return columnSplit
- *
- * @return
- */
- public int[] getColumnSplit() {
- return columnSplit;
- }
-
- /**
- * set columnSplit
- *
- * @param split
- */
- public void setColumnSplit(int[] split) {
- this.columnSplit = split;
- }
-
- /**
- * @return no of columnar block
- */
- public int getNoOfColumnStore() {
- return this.noOfColumnsStore;
- }
-
- /**
- * set no of columnar block
- *
- * @param noOfColumnsStore
- */
- public void setNoOfColumnStore(int noOfColumnsStore) {
- this.noOfColumnsStore = noOfColumnsStore;
- }
-
- /**
- * set column groups
- *
- * @param columnGroups
- */
- public void setColumnGroup(int[][] columnGroups) {
- this.columnGroups = columnGroups;
- }
-
- /**
- * @return columngroups
- */
- public int[][] getColumnGroup() {
- return this.columnGroups;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
index dd5d08d..5cd8f84 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
@@ -39,26 +39,12 @@ public class ColumnWithRowId<T> implements Comparable<ColumnWithRowId<T>> {
}
/**
- * @param column the column to set
- */
- public void setColumn(byte[] column) {
- this.column = column;
- }
-
- /**
* @return the index
*/
public T getIndex() {
return index;
}
- /**
- * @param index the index to set
- */
- public void setIndex(T index) {
- this.index = index;
- }
-
@Override public int compareTo(ColumnWithRowId o) {
return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/page/EncodedTablePage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/EncodedTablePage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/EncodedTablePage.java
index dc0a388..1770f72 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/EncodedTablePage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/EncodedTablePage.java
@@ -42,24 +42,12 @@ public class EncodedTablePage {
// size in bytes of all encoded columns (including data and metadate)
private int encodedSize;
- public static EncodedTablePage newEmptyInstance() {
- EncodedTablePage page = new EncodedTablePage();
- page.pageSize = 0;
- page.encodedSize = 0;
- page.dimensionPages = new EncodedColumnPage[0];
- page.measurePages = new EncodedColumnPage[0];
- return page;
- }
-
public static EncodedTablePage newInstance(int pageSize,
EncodedColumnPage[] dimensionPages, EncodedColumnPage[] measurePages,
TablePageKey tablePageKey) throws IOException {
return new EncodedTablePage(pageSize, dimensionPages, measurePages, tablePageKey);
}
- private EncodedTablePage() {
- }
-
private EncodedTablePage(int pageSize,
EncodedColumnPage[] dimensionPages, EncodedColumnPage[] measurePages,
TablePageKey tablePageKey) throws IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
index dc79b13..fa8bfad 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
@@ -253,13 +253,8 @@ public class DefaultEncodingFactory extends EncodingFactory {
boolean isComplexPrimitive) {
DataType srcDataType = stats.getDataType();
DataType adaptiveDataType = fitMinMax(stats.getDataType(), stats.getMax(), stats.getMin());
- DataType deltaDataType;
- if (adaptiveDataType == DataTypes.LONG) {
- deltaDataType = DataTypes.LONG;
- } else {
- deltaDataType = fitDelta(stats.getDataType(), stats.getMax(), stats.getMin());
- }
+ DataType deltaDataType = fitDelta(stats.getDataType(), stats.getMax(), stats.getMin());
// for complex primitive, if source and destination data type is same, use adaptive encoding.
if (!isComplexPrimitive) {
// in case of decimal datatype, check if the decimal converter type is Int or Long and based
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index 7adeff0..7ac908f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -27,7 +27,6 @@ import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPageValueConverter;
import org.apache.carbondata.core.datastore.page.LazyColumnPage;
-import org.apache.carbondata.core.datastore.page.encoding.ColumnPageCodec;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
@@ -47,11 +46,6 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
private ColumnPage encodedPage;
private Double factor;
- public static ColumnPageCodec newInstance(DataType srcDataType, DataType targetDataType,
- SimpleStatsResult stats) {
- return new AdaptiveFloatingCodec(srcDataType, targetDataType, stats);
- }
-
public AdaptiveFloatingCodec(DataType srcDataType, DataType targetDataType,
SimpleStatsResult stats) {
super(srcDataType, targetDataType, stats);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java
index edc3b64..37d9052 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java
@@ -38,11 +38,7 @@ public abstract class IndexStorageEncoder extends ColumnPageEncoder {
@Override
protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException {
- assert (indexStorage == null);
- assert (compressedDataPage == null);
encodeIndexStorage(input);
- assert (indexStorage != null);
- assert (compressedDataPage != null);
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
out.write(compressedDataPage);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
index 244d8fc..c2b722b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
@@ -74,10 +74,6 @@ public class TablePageKey {
}
}
- public Object getKey() {
- return this;
- }
-
/** update all keys if SORT_COLUMNS option is used when creating table */
private void finalizeKeys() {
// If SORT_COLUMNS is used, may need to update start/end keys since the they may
@@ -131,14 +127,6 @@ public class TablePageKey {
}
}
- public byte[] getStartKey() {
- return startKey;
- }
-
- public byte[] getEndKey() {
- return endKey;
- }
-
public byte[] getNoDictStartKey() {
return packedNoDictStartKey;
}
@@ -147,10 +135,6 @@ public class TablePageKey {
return packedNoDictEndKey;
}
- public int getPageSize() {
- return pageSize;
- }
-
public byte[] serializeStartKey() {
byte[] updatedNoDictionaryStartKey = updateNoDictionaryStartAndEndKey(getNoDictStartKey());
ByteBuffer buffer = ByteBuffer.allocate(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
index 82f004f..48775d4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
@@ -61,14 +61,6 @@ public class CarbonRow implements Serializable {
return data[ordinal];
}
- public Object[] getObjectArray(int ordinal) {
- return (Object[]) data[ordinal];
- }
-
- public int[] getIntArray(int ordinal) {
- return (int[]) data[ordinal];
- }
-
public void update(Object value, int ordinal) {
data[ordinal] = value;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/devapi/DictionaryGenerationException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/devapi/DictionaryGenerationException.java b/core/src/main/java/org/apache/carbondata/core/devapi/DictionaryGenerationException.java
index fa84c0a..6de9d9c 100644
--- a/core/src/main/java/org/apache/carbondata/core/devapi/DictionaryGenerationException.java
+++ b/core/src/main/java/org/apache/carbondata/core/devapi/DictionaryGenerationException.java
@@ -16,8 +16,6 @@
*/
package org.apache.carbondata.core.devapi;
-import java.util.Locale;
-
public class DictionaryGenerationException extends Exception {
/**
* default serial version ID.
@@ -25,61 +23,11 @@ public class DictionaryGenerationException extends Exception {
private static final long serialVersionUID = 1L;
/**
- * The Error message.
- */
- private String msg = "";
-
- /**
* Constructor
*
* @param msg The error message for this exception.
*/
public DictionaryGenerationException(String msg) {
super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public DictionaryGenerationException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param t
- */
- public DictionaryGenerationException(Throwable t) {
- super(t);
- }
-
- /**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/devapi/GeneratingBiDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/devapi/GeneratingBiDictionary.java b/core/src/main/java/org/apache/carbondata/core/devapi/GeneratingBiDictionary.java
deleted file mode 100644
index 7d961f8..0000000
--- a/core/src/main/java/org/apache/carbondata/core/devapi/GeneratingBiDictionary.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.devapi;
-
-public abstract class GeneratingBiDictionary<K, V> implements BiDictionary<K, V> {
-
- private DictionaryGenerator<K, V> generator;
-
- public GeneratingBiDictionary(DictionaryGenerator<K, V> generator) {
- this.generator = generator;
- }
-
- @Override
- public K getOrGenerateKey(V value) throws DictionaryGenerationException {
- K key = getKey(value);
- if (key != null) {
- return key;
- } else {
- K newKey = generator.generateKey(value);
- assert (newKey != null);
- put(newKey, value);
- return newKey;
- }
- }
-
- /**
- * put the input key value pair into the dictionary
- * @param key dictionary key
- * @param value dictionary value
- */
- protected abstract void put(K key, V value);
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
index 64e6252..fce0338 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
@@ -21,9 +21,9 @@ import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
public interface DictionaryClient {
- public void startClient(String secretKey, String address, int port, boolean encryptSecureServer);
+ void startClient(String secretKey, String address, int port, boolean encryptSecureServer);
- public void shutDown();
+ void shutDown();
- public DictionaryMessage getDictionary(DictionaryMessage key);
+ DictionaryMessage getDictionary(DictionaryMessage key);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
index 7bb8259..3dbe5b0 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
@@ -122,8 +122,4 @@ public class TableDictionaryGenerator
}
}
}
- public String getTableUniqueName() {
- return carbonTable.getTableUniqueName();
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
index 3b3f21a..72164a1 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
@@ -183,9 +183,4 @@ public class DictionaryMessage {
public void setTableUniqueId(String tableUniqueId) {
this.tableUniqueId = tableUniqueId;
}
-
- @Override public String toString() {
- return "DictionaryKey{ columnName='" + columnName + '\'' + ", data='" + data + '\''
- + ", dictionaryValue=" + dictionaryValue + ", type=" + type + '}';
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
index a23cc9b..f814281 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
@@ -21,21 +21,21 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
public interface DictionaryServer {
- public void startServer();
+ void startServer();
- public void bindToPort();
+ void bindToPort();
- public void shutdown()throws Exception;
+ void shutdown()throws Exception;
- public String getHost();
+ String getHost();
- public int getPort();
+ int getPort();
- public String getSecretKey();
+ String getSecretKey();
- public boolean isEncryptSecureServer();
+ boolean isEncryptSecureServer();
- public void writeTableDictionary(String uniqueTableName) throws Exception;
+ void writeTableDictionary(String uniqueTableName) throws Exception;
- public void initializeDictionaryGenerator(CarbonTable carbonTable) throws Exception;
+ void initializeDictionaryGenerator(CarbonTable carbonTable) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryServiceProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryServiceProvider.java b/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryServiceProvider.java
index 4dfa1e9..5d7d813 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryServiceProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/service/DictionaryServiceProvider.java
@@ -23,5 +23,5 @@ import org.apache.carbondata.core.dictionary.client.DictionaryClient;
public interface DictionaryServiceProvider extends Serializable {
- public DictionaryClient getDictionaryClient();
+ DictionaryClient getDictionaryClient();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/exception/InvalidConfigurationException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/exception/InvalidConfigurationException.java b/core/src/main/java/org/apache/carbondata/core/exception/InvalidConfigurationException.java
index bef9576..08b03a8 100644
--- a/core/src/main/java/org/apache/carbondata/core/exception/InvalidConfigurationException.java
+++ b/core/src/main/java/org/apache/carbondata/core/exception/InvalidConfigurationException.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.core.exception;
-import java.util.Locale;
-
public class InvalidConfigurationException extends Exception {
/**
@@ -52,33 +50,6 @@ public class InvalidConfigurationException extends Exception {
}
/**
- * Constructor
- *
- * @param t
- */
- public InvalidConfigurationException(Throwable t) {
- super(t);
- }
-
- /**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
* getMessage
*/
public String getMessage() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java
index 971f42a..7f47c00 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/CarbonRowSchema.java
@@ -51,7 +51,9 @@ public abstract class CarbonRowSchema implements Serializable {
*
* @return
*/
- public abstract int getLength();
+ public int getLength() {
+ return dataType.getSizeInBytes();
+ }
/**
* schema type
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java
index 09e162c..b083ee0 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/KeyGenerator.java
@@ -80,11 +80,6 @@ public interface KeyGenerator extends Serializable, Comparator<byte[]> {
long getKey(byte[] key, int index);
/**
- * Set any extra properties if required.
- */
- void setProperty(Object key, Object value);
-
- /**
* Gives the key size in number of bytes.
*/
int getKeySizeInBytes();
@@ -98,8 +93,6 @@ public interface KeyGenerator extends Serializable, Comparator<byte[]> {
*/
int[] getKeyByteOffsets(int index);
- int compare(byte[] key1, int offset1, int length1, byte[] key2, int offset2, int length2);
-
/**
* returns the dimension count
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
index 9ce6748..4092342 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
@@ -30,7 +30,6 @@ public final class DirectDictionaryKeyGeneratorFactory {
* private constructor
*/
private DirectDictionaryKeyGeneratorFactory() {
-
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java
index a480595..86eb4e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/AbstractKeyGenerator.java
@@ -40,34 +40,4 @@ public abstract class AbstractKeyGenerator implements KeyGenerator {
}
return 0;
}
-
- public int compare(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2,
- int length2) {
- length1 += offset1;
- length2 += offset2;
- // Bring WritableComparator code local
- for (; offset1 < length1 && offset2 < length2; offset1++, offset2++) {
- int a = (buffer1[offset1] & 0xff);
- int b = (buffer2[offset2] & 0xff);
- if (a != b) {
- return a - b;
- }
- }
- return 0;
- }
-
- @Override public void setProperty(Object key, Object value) {
- /*
- * No implementation required.
- */
- }
-
- @Override public int getKeySizeInBytes() {
- return 0;
- }
-
- @Override public int getDimCount() {
- return 0;
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
index d601159..be680e9 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
@@ -103,40 +103,6 @@ public class NumberCompressor {
return words;
}
- protected long[] get(byte[] keys, int wsize) {
- long[] words = new long[wsize];
- int ll = 0;
- long val = 0L;
- for (int i = keys.length - 1; i >= 0; ) {
-
- int size = i;
- val = 0L;
- for (int j = i + 1; j <= size; ) {
- val <<= BYTE_LENGTH;
- val ^= keys[j++] & 0xFF;
- i--;
- }
- int index = ll >> 6;// divide by 64 to get the new word index
- words[index] |= (val << (ll & 0x3f));
- ll += bitsLength;
-
- int nextIndex = ll >> 6;// This is divide by 64
-
- if (nextIndex != index) {
- int consideredBits = bitsLength - ll & 0x3f;
- if (consideredBits < bitsLength) // Check for spill over only if
- // all the bits are not
- // considered
- {
- // Check for spill over
- words[nextIndex] |= (val >> (bitsLength - ll & 0x3f));
- }
- }
-
- }
- return words;
- }
-
public int[] unCompress(byte[] key, int offset, int length) {
int arrayLength = (length * BYTE_LENGTH) / bitsLength;
long[] words = new long[getWordsSizeFromBytesSize(length)];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
index 95bc932..4d67faf 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
@@ -128,9 +128,7 @@ public class CarbonLockUtil {
@Override public boolean accept(CarbonFile pathName) {
if (CarbonTablePath.isSegmentLockFilePath(pathName.getName())) {
- if ((currTime - pathName.getLastModifiedTime()) > segmentLockFilesPreservTime) {
- return true;
- }
+ return (currTime - pathName.getLastModifiedTime()) > segmentLockFilesPreservTime;
}
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/locks/ZookeeperInit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/ZookeeperInit.java b/core/src/main/java/org/apache/carbondata/core/locks/ZookeeperInit.java
index ecdb672..102ea4c 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/ZookeeperInit.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/ZookeeperInit.java
@@ -64,10 +64,6 @@ public class ZookeeperInit {
}
- public static ZookeeperInit getInstance() {
- return zooKeeperInit;
- }
-
public ZooKeeper getZookeeper() {
return zk;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
index 242995b..d898dc9 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
@@ -87,7 +87,6 @@ public class HeapMemoryAllocator implements MemoryAllocator {
// As an additional layer of defense against use-after-free bugs, we mutate the
// MemoryBlock to null out its reference to the long[] array.
long[] array = (long[]) memory.obj;
- memory.setObjAndOffset(null, 0);
long alignedSize = ((size + 7) / 8) * 8;
if (shouldPool(alignedSize)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java b/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java
index fd4f06c..418ef89 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/MemoryBlock.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.memory;
import javax.annotation.Nullable;
+
/**
* Code ported from Apache Spark {org.apache.spark.unsafe.memory} package
* A consecutive block of memory, starting at a {@link MemoryLocation} with a fixed size.
@@ -52,5 +53,4 @@ public class MemoryBlock extends MemoryLocation {
public void setFreedStatus(boolean freedStatus) {
this.isFreed = freedStatus;
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/memory/MemoryLocation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/MemoryLocation.java b/core/src/main/java/org/apache/carbondata/core/memory/MemoryLocation.java
index 0419e04..c059377 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/MemoryLocation.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/MemoryLocation.java
@@ -36,15 +36,6 @@ public class MemoryLocation {
this.offset = offset;
}
- public MemoryLocation() {
- this(null, 0);
- }
-
- public void setObjAndOffset(Object newObj, long newOffset) {
- this.obj = newObj;
- this.offset = newOffset;
- }
-
public final Object getBaseObject() {
return obj;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/compressor/ChunkCompressorMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/compressor/ChunkCompressorMeta.java b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/compressor/ChunkCompressorMeta.java
deleted file mode 100644
index 55f7050..0000000
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/compressor/ChunkCompressorMeta.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.metadata.blocklet.compressor;
-
-import java.io.Serializable;
-
-/**
- * Represents the compression information of data of one dimension
- * one dimension group in one blocklet
- */
-public class ChunkCompressorMeta implements Serializable {
-
- /**
- * serialization version
- */
- private static final long serialVersionUID = -6697087170420991140L;
-
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 8206fef..aa27a81 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -179,10 +179,10 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperColumnSchema.getDataType()),
wrapperColumnSchema.getColumnName(),
wrapperColumnSchema.getColumnUniqueId(),
- wrapperColumnSchema.isColumnar(),
+ true,
encoders,
wrapperColumnSchema.isDimensionColumn());
- thriftColumnSchema.setColumn_group_id(wrapperColumnSchema.getColumnGroupId());
+ thriftColumnSchema.setColumn_group_id(-1);
if (DataTypes.isDecimal(wrapperColumnSchema.getDataType())) {
thriftColumnSchema.setScale(wrapperColumnSchema.getScale());
thriftColumnSchema.setPrecision(wrapperColumnSchema.getPrecision());
@@ -511,7 +511,6 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
ColumnSchema wrapperColumnSchema = new ColumnSchema();
wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
- wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
wrapperColumnSchema.setDataType(
fromExternalToWrapperDataType(
externalColumnSchema.data_type,
@@ -525,7 +524,6 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperColumnSchema.setEncodingList(encoders);
wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());
wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());
- wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
wrapperColumnSchema.setScale(externalColumnSchema.getScale());
wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
index e4059c8..a8da6d4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
@@ -104,20 +104,6 @@ public final class DecimalConverterFactory {
}
}
- private long getUnscaledLong(byte[] bytes) {
- long unscaled = 0L;
- int i = 0;
-
- while (i < bytes.length) {
- unscaled = (unscaled << 8) | (bytes[i] & 0xff);
- i += 1;
- }
-
- int bits = 8 * bytes.length;
- unscaled = (unscaled << (64 - bits)) >> (64 - bits);
- return unscaled;
- }
-
public static class DecimalLongConverter implements DecimalConverter {
private int scale;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 086cedd..a6936b6 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -322,8 +322,6 @@ public class CarbonTable implements Serializable {
int dimensionOrdinal = 0;
int measureOrdinal = 0;
int keyOrdinal = 0;
- int columnGroupOrdinal = -1;
- int previousColumnGroupId = -1;
List<ColumnSchema> listOfColumns = tableSchema.getListOfColumns();
int complexTypeOrdinal = -1;
for (int i = 0; i < listOfColumns.size(); i++) {
@@ -331,8 +329,8 @@ public class CarbonTable implements Serializable {
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(),
- -1, -1, ++complexTypeOrdinal);
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, ++complexTypeOrdinal);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
allDimensions.add(complexDimension);
dimensionOrdinal =
@@ -345,27 +343,18 @@ public class CarbonTable implements Serializable {
this.numberOfSortColumns++;
}
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
- CarbonDimension dimension = new CarbonDimension(
- columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ CarbonDimension dimension =
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, -1);
if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
this.numberOfNoDictSortColumns++;
}
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
- } else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
- && columnSchema.getColumnGroupId() == -1) {
- CarbonDimension dimension = new CarbonDimension(
- columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), keyOrdinal++,
- -1, -1);
- allDimensions.add(dimension);
- primitiveDimensions.add(dimension);
- } else {
- columnGroupOrdinal =
- previousColumnGroupId == columnSchema.getColumnGroupId() ? ++columnGroupOrdinal : 0;
- previousColumnGroupId = columnSchema.getColumnGroupId();
- CarbonDimension dimension = new CarbonDimension(
- columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), keyOrdinal++,
- columnGroupOrdinal, -1);
+ } else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
+ CarbonDimension dimension =
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), keyOrdinal++, -1);
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
}
@@ -425,8 +414,8 @@ public class CarbonTable implements Serializable {
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(),
- -1, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, -1);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
parentDimension.getListOfChildDimensions().add(complexDimension);
dimensionOrdinal =
@@ -434,8 +423,8 @@ public class CarbonTable implements Serializable {
listOfColumns, complexDimension, primitiveDimensions);
} else {
CarbonDimension carbonDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(),
- -1, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, -1);
parentDimension.getListOfChildDimensions().add(carbonDimension);
primitiveDimensions.add(carbonDimension);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
index e760da4..4780e60 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
@@ -183,7 +183,6 @@ public class TableSchemaBuilder {
// child column should not be counted for schema ordinal
newColumn.setSchemaOrdinal(-1);
}
- newColumn.setColumnar(true);
// For NonTransactionalTable, multiple sdk writer output with same column name can be placed in
// single folder for query.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
index e19e329..809cc30 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
@@ -74,12 +74,6 @@ public class CarbonColumn implements Serializable {
this.columnIdentifier =
new ColumnIdentifier(getColumnId(), getColumnProperties(), getDataType());
}
- /**
- * @return columnar or row based
- */
- public boolean isColumnar() {
- return columnSchema.isColumnar();
- }
/**
* @return column unique id
@@ -117,13 +111,6 @@ public class CarbonColumn implements Serializable {
}
/**
- * @return row group id if it is row based
- */
- public int columnGroupId() {
- return columnSchema.getColumnGroupId();
- }
-
- /**
* @return the defaultValue
*/
public byte[] getDefaultValue() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
index bec9889..f0ff1c8 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
@@ -40,13 +40,6 @@ public class CarbonDimension extends CarbonColumn {
private int keyOrdinal;
/**
- * column group column ordinal
- * for example if column is second column in the group
- * it will store 2
- */
- private int columnGroupOrdinal;
-
- /**
* to store complex type dimension ordinal
*/
private int complexTypeOrdinal;
@@ -57,15 +50,14 @@ public class CarbonDimension extends CarbonColumn {
private CarbonDimension complexParentDimension = null;
public CarbonDimension(ColumnSchema columnSchema, int ordinal, int keyOrdinal,
- int columnGroupOrdinal, int complexTypeOrdinal) {
- this(columnSchema, ordinal, 0, keyOrdinal, columnGroupOrdinal, complexTypeOrdinal);
+ int complexTypeOrdinal) {
+ this(columnSchema, ordinal, 0, keyOrdinal, complexTypeOrdinal);
}
public CarbonDimension(ColumnSchema columnSchema, int ordinal, int schemaOrdinal, int keyOrdinal,
- int columnGroupOrdinal, int complexTypeOrdinal) {
+ int complexTypeOrdinal) {
super(columnSchema, ordinal, schemaOrdinal);
this.keyOrdinal = keyOrdinal;
- this.columnGroupOrdinal = columnGroupOrdinal;
this.complexTypeOrdinal = complexTypeOrdinal;
}
@@ -102,13 +94,6 @@ public class CarbonDimension extends CarbonColumn {
}
/**
- * @return the columnGroupOrdinal
- */
- public int getColumnGroupOrdinal() {
- return columnGroupOrdinal;
- }
-
- /**
* @return the complexTypeOrdinal
*/
public int getComplexTypeOrdinal() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonImplicitDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonImplicitDimension.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonImplicitDimension.java
index 9e0cc22..fadbd02 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonImplicitDimension.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonImplicitDimension.java
@@ -47,7 +47,7 @@ public class CarbonImplicitDimension extends CarbonDimension {
private String implicitDimensionName;
public CarbonImplicitDimension(int ordinal, String implicitDimensionName) {
- super(null, ordinal, -1, -1, -1);
+ super(null, ordinal, -1, -1);
encodingList = new ArrayList<Encoding>();
encodingList.add(Encoding.IMPLICIT);
this.implicitDimensionName = implicitDimensionName;
@@ -77,13 +77,6 @@ public class CarbonImplicitDimension extends CarbonDimension {
}
/**
- * @return row group id if it is row based
- */
- @Override public int columnGroupId() {
- return -1;
- }
-
- /**
* @return the list of encoder used in dimension
*/
@Override public List<Encoding> getEncoder() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 9bb9e85..853851d 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -84,12 +84,6 @@ public class ColumnSchema implements Serializable, Writable {
private boolean isDimensionColumn;
/**
- * The group ID for column used for row format columns,
- * where in columns in each group are chunked together.
- */
- private int columnGroupId = -1;
-
- /**
* Used when this column contains decimal data.
*/
private int scale;
@@ -185,20 +179,6 @@ public class ColumnSchema implements Serializable, Writable {
}
/**
- * @return the isColumnar
- */
- public boolean isColumnar() {
- return isColumnar;
- }
-
- /**
- * @param isColumnar the isColumnar to set
- */
- public void setColumnar(boolean isColumnar) {
- this.isColumnar = isColumnar;
- }
-
- /**
* @return the isDimensionColumn
*/
public boolean isDimensionColumn() {
@@ -235,19 +215,6 @@ public class ColumnSchema implements Serializable, Writable {
}
/**
- * @return the columnGroup
- */
- public int getColumnGroupId() {
- return columnGroupId;
- }
-
- /**
- */
- public void setColumnGroup(int columnGroupId) {
- this.columnGroupId = columnGroupId;
- }
-
- /**
* Set the scale if it is decimal type
*/
public void setScale(int scale) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 8627bdb..d0a204c 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -257,7 +257,7 @@ public class CarbonUpdateUtil {
}
try {
- segmentStatusManager
+ SegmentStatusManager
.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetailsArray);
} catch (IOException e) {
return false;
@@ -469,7 +469,7 @@ public class CarbonUpdateUtil {
String segmentPath = CarbonTablePath.getSegmentPath(
table.getAbsoluteTableIdentifier().getTablePath(), segment.getLoadName());
CarbonFile segDir =
- FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
+ FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
// scan through the segment and find the carbondatafiles and index files.
@@ -557,7 +557,6 @@ public class CarbonUpdateUtil {
}
}
-
} else {
invalidDeleteDeltaFiles = updateStatusManager
.getDeleteDeltaInvalidFilesList(block, false,
@@ -594,7 +593,7 @@ public class CarbonUpdateUtil {
if (null != validUpdateStatusFile && !validUpdateStatusFile.isEmpty()) {
final String updateStatusTimestamp = validUpdateStatusFile
- .substring(validUpdateStatusFile.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1);
+ .substring(validUpdateStatusFile.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1);
String tablePath = table.getAbsoluteTableIdentifier().getTablePath();
CarbonFile metaFolder = FileFactory.getCarbonFile(
@@ -607,9 +606,7 @@ public class CarbonUpdateUtil {
// CHECK if this is valid or not.
// we only send invalid ones to delete.
- if (!file.getName().endsWith(updateStatusTimestamp)) {
- return true;
- }
+ return !file.getName().endsWith(updateStatusTimestamp);
}
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/mutate/data/DeleteDeltaCacheLoaderIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/data/DeleteDeltaCacheLoaderIntf.java b/core/src/main/java/org/apache/carbondata/core/mutate/data/DeleteDeltaCacheLoaderIntf.java
deleted file mode 100644
index 924e03a..0000000
--- a/core/src/main/java/org/apache/carbondata/core/mutate/data/DeleteDeltaCacheLoaderIntf.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.mutate.data;
-
-/**
- * This interface holds all methods required to load delete delta file data to cache
- */
-public interface DeleteDeltaCacheLoaderIntf {
-
- void loadDeleteDeltaFileDataToCache();
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/readcommitter/ReadCommittedScope.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/readcommitter/ReadCommittedScope.java b/core/src/main/java/org/apache/carbondata/core/readcommitter/ReadCommittedScope.java
index d177a00..cbcf173 100644
--- a/core/src/main/java/org/apache/carbondata/core/readcommitter/ReadCommittedScope.java
+++ b/core/src/main/java/org/apache/carbondata/core/readcommitter/ReadCommittedScope.java
@@ -31,22 +31,21 @@ import org.apache.carbondata.core.statusmanager.SegmentRefreshInfo;
* ReadCommitted interface that defines a read scope.
*/
@InterfaceAudience.Internal
-@InterfaceStability.Stable
-public interface ReadCommittedScope extends Serializable {
+@InterfaceStability.Stable public interface ReadCommittedScope extends Serializable {
- public LoadMetadataDetails[] getSegmentList() throws IOException;
+ LoadMetadataDetails[] getSegmentList() throws IOException;
/**
* @param segment
* @return map of Absolute path of index file as key and null as value -- without mergeIndex
* map of AbsolutePath with fileName of MergeIndex parent file as key and mergeIndexFileName
- * as value -- with mergeIndex
+ * as value -- with mergeIndex
* @throws IOException
*/
- public Map<String, String> getCommittedIndexFile(Segment segment) throws IOException ;
+ Map<String, String> getCommittedIndexFile(Segment segment) throws IOException;
- public SegmentRefreshInfo getCommittedSegmentRefreshInfo(
- Segment segment, UpdateVO updateVo) throws IOException;
+ SegmentRefreshInfo getCommittedSegmentRefreshInfo(Segment segment, UpdateVO updateVo)
+ throws IOException;
- public void takeCarbonIndexFileSnapShot() throws IOException;
+ void takeCarbonIndexFileSnapShot() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
index 8902fc2..ea4afd1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/ResultCollectorFactory.java
@@ -82,4 +82,7 @@ public class ResultCollectorFactory {
}
return scannerResultAggregator;
}
+
+ private ResultCollectorFactory() {
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/executor/exception/QueryExecutionException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/exception/QueryExecutionException.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/exception/QueryExecutionException.java
index 8e25ff4..1d76de6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/exception/QueryExecutionException.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/exception/QueryExecutionException.java
@@ -16,8 +16,6 @@
*/
package org.apache.carbondata.core.scan.executor.exception;
-import java.util.Locale;
-
/**
* Exception class for query execution
*/
@@ -29,31 +27,6 @@ public class QueryExecutionException extends Exception {
private static final long serialVersionUID = 1L;
/**
- * The Error message.
- */
- private String msg = "";
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public QueryExecutionException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public QueryExecutionException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
* Constructor
*
* @param t
@@ -63,28 +36,9 @@ public class QueryExecutionException extends Exception {
}
/**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
* getLocalizedMessage
*/
@Override public String getLocalizedMessage() {
return super.getLocalizedMessage();
}
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 7e116d9..180ca4d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -396,9 +396,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
blockExecutionInfo.setStartKey(startIndexKey);
//setting the end index key of the block node
blockExecutionInfo.setEndKey(endIndexKey);
- // expression dimensions
- List<CarbonDimension> expressionDimensions =
- new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
// expression measure
List<CarbonMeasure> expressionMeasures =
new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
@@ -409,9 +406,9 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// create a list of filter dimensions present in the current block
Set<CarbonDimension> currentBlockFilterDimensions =
getCurrentBlockFilterDimensions(queryProperties.complexFilterDimension, segmentProperties);
- int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(
- projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(),
- expressionDimensions, currentBlockFilterDimensions, allProjectionListDimensionIdexes);
+ int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(projectDimensions,
+ segmentProperties.getDimensionOrdinalToChunkMapping(),
+ currentBlockFilterDimensions, allProjectionListDimensionIdexes);
int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE));
@@ -482,16 +479,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
blockExecutionInfo.setComplexColumnParentBlockIndexes(
getComplexDimensionParentBlockIndexes(projectDimensions));
blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
- try {
- // to set column group and its key structure info which will be used
- // to
- // for getting the column group column data in case of final row
- // and in case of dimension aggregation
- blockExecutionInfo.setColumnGroupToKeyStructureInfo(
- QueryUtil.getColumnGroupKeyStructureInfo(projectDimensions, segmentProperties));
- } catch (KeyGenException e) {
- throw new QueryExecutionException(e);
- }
// set actual query dimensions and measures. It may differ in case of restructure scenarios
blockExecutionInfo.setActualQueryDimensions(queryModel.getProjectionDimensions()
.toArray(new ProjectionDimension[queryModel.getProjectionDimensions().size()]));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index 3165bfd..e737b0e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -121,11 +121,6 @@ public class BlockExecutionInfo {
private int[] eachColumnValueSize;
/**
- * column group block index in file to key structure info mapping
- */
- private Map<Integer, KeyStructureInfo> columnGroupToKeyStructureInfo;
-
- /**
* filter tree to execute the filter
*/
private FilterExecuter filterExecuterTree;
@@ -425,21 +420,6 @@ public class BlockExecutionInfo {
this.noDictionaryColumnChunkIndexes = noDictionaryColumnChunkIndexes;
}
- /**
- * @return the columnGroupToKeyStructureInfo
- */
- public Map<Integer, KeyStructureInfo> getColumnGroupToKeyStructureInfo() {
- return columnGroupToKeyStructureInfo;
- }
-
- /**
- * @param columnGroupToKeyStructureInfo the columnGroupToKeyStructureInfo to set
- */
- public void setColumnGroupToKeyStructureInfo(
- Map<Integer, KeyStructureInfo> columnGroupToKeyStructureInfo) {
- this.columnGroupToKeyStructureInfo = columnGroupToKeyStructureInfo;
- }
-
public boolean isRawRecordDetailQuery() {
return isRawRecordDetailQuery;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9114036/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/KeyStructureInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/KeyStructureInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/KeyStructureInfo.java
deleted file mode 100644
index 0cccee4..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/KeyStructureInfo.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.executor.infos;
-
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-
-/**
- * Below class will store the structure of the key
- * used during query execution
- */
-public class KeyStructureInfo {
-
- /**
- * it's actually a latest key generator
- * last table block as this key generator will be used to
- * to update the mdkey of the older slice with the new slice
- */
- private KeyGenerator keyGenerator;
-
- /**
- * mask bytes ranges for the query
- */
- private int[] maskByteRanges;
-
- /**
- * max key for query execution
- */
- private byte[] maxKey;
-
- /**
- * dimension ordinals inside the column group
- */
- private int[] mdkeyQueryDimensionOrdinal;
-
- /**
- * @return the keyGenerator
- */
- public KeyGenerator getKeyGenerator() {
- return keyGenerator;
- }
-
- /**
- * @param keyGenerator the keyGenerator to set
- */
- public void setKeyGenerator(KeyGenerator keyGenerator) {
- this.keyGenerator = keyGenerator;
- }
-
- /**
- * @return the maskByteRanges
- */
- public int[] getMaskByteRanges() {
- return maskByteRanges;
- }
-
- /**
- * @param maskByteRanges the maskByteRanges to set
- */
- public void setMaskByteRanges(int[] maskByteRanges) {
- this.maskByteRanges = maskByteRanges;
- }
-
- /**
- * @return the maxKey
- */
- public byte[] getMaxKey() {
- return maxKey;
- }
-
- /**
- * @param maxKey the maxKey to set
- */
- public void setMaxKey(byte[] maxKey) {
- this.maxKey = maxKey;
- }
-
- public int[] getMdkeyQueryDimensionOrdinal() {
- return mdkeyQueryDimensionOrdinal;
- }
-
- public void setMdkeyQueryDimensionOrdinal(int[] mdkeyQueryDimensionOrdinal) {
- this.mdkeyQueryDimensionOrdinal = mdkeyQueryDimensionOrdinal;
- }
-}