You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2018/02/27 08:18:58 UTC
[01/16] carbondata git commit: [CARBONDATA-2161] update mergeTo
column for compacted segment of streaming table
Repository: carbondata
Updated Branches:
refs/heads/branch-1.3 39ac94e46 -> 1997ca235
[CARBONDATA-2161] update mergeTo column for compacted segment of streaming table
This closes #1971
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c2785b35
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c2785b35
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c2785b35
Branch: refs/heads/branch-1.3
Commit: c2785b352f7b7cb2dd524811b0696fb18c12d5b0
Parents: 39ac94e
Author: BJangir <ba...@gmail.com>
Authored: Mon Feb 12 01:02:30 2018 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Tue Feb 27 12:45:55 2018 +0530
----------------------------------------------------------------------
.../apache/spark/carbondata/TestStreamingTableOperation.scala | 5 +++++
.../org/apache/carbondata/streaming/StreamHandoffRDD.scala | 1 +
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c2785b35/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index a368cef..4b3a957 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -637,6 +637,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
assertResult(newSegments.length / 2)(newSegments.filter(_.getString(1).equals("Success")).length)
assertResult(newSegments.length / 2)(newSegments.filter(_.getString(1).equals("Compacted")).length)
+ //Verify MergeTO column entry for compacted Segments
+ newSegments.filter(_.getString(1).equals("Compacted")).foreach{ rw =>
+ assertResult("Compacted")(rw.getString(1))
+ assertResult((Integer.parseInt(rw.getString(0))+2).toString)(rw.getString(4))
+ }
checkAnswer(
sql("select count(*) from streaming.stream_table_reopen"),
Seq(Row(2 * 100 * 2))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c2785b35/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 41dfa50..4caa401 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -398,6 +398,7 @@ object StreamHandoffRDD {
throw new Exception("Failed to update table status for streaming segment")
} else {
streamSegment.get.setSegmentStatus(SegmentStatus.COMPACTED)
+ streamSegment.get.setMergedLoadName(loadModel.getSegmentId)
}
// refresh table status file
[14/16] carbondata git commit: [CARBONDATA-2187][PARTITION] Partition
restructure for new folder structure and supporting partition location
feature
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 984efdb..f4450e3 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -29,6 +29,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
@@ -714,15 +715,15 @@ public class CarbonUtilTest {
}
@Test public void testToGetSegmentString() {
- List<String> list = new ArrayList<>();
- list.add("1");
- list.add("2");
+ List<Segment> list = new ArrayList<>();
+ list.add(new Segment("1", null));
+ list.add(new Segment("2", null));
String segments = CarbonUtil.convertToString(list);
assertEquals(segments, "1,2");
}
@Test public void testToGetSegmentStringWithEmptySegmentList() {
- List<String> list = new ArrayList<>();
+ List<Segment> list = new ArrayList<>();
String segments = CarbonUtil.convertToString(list);
assertEquals(segments, "");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 5ab6605..a4c6e4a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.block.BlockletInfos;
import org.apache.carbondata.core.datastore.block.Distributable;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -260,8 +261,12 @@ public class CarbonInputSplit extends FileSplit
return invalidSegments;
}
- public void setInvalidSegments(List<String> invalidSegments) {
- this.invalidSegments = invalidSegments;
+ public void setInvalidSegments(List<Segment> invalidSegments) {
+ List<String> invalidSegmentIds = new ArrayList<>();
+ for (Segment segment: invalidSegments) {
+ invalidSegmentIds.add(segment.getSegmentNo());
+ }
+ this.invalidSegments = invalidSegmentIds;
}
public void setInvalidTimestampRange(List<UpdateVO> invalidTimestamps) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index ce97169..7ea11bd 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -18,11 +18,22 @@
package org.apache.carbondata.hadoop.api;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.PartitionMapFileStore;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
@@ -52,6 +63,8 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(CarbonOutputCommitter.class.getName());
+ private ICarbonLock segmentLock;
+
public CarbonOutputCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
super(outputPath, context);
}
@@ -66,9 +79,16 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
super.setupJob(context);
boolean overwriteSet = CarbonTableOutputFormat.isOverwriteSet(context.getConfiguration());
CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
- CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(loadModel, overwriteSet);
- CarbonLoaderUtil.checkAndCreateCarbonDataLocation(loadModel.getSegmentId(),
- loadModel.getCarbonDataLoadSchema().getCarbonTable());
+ if (loadModel.getSegmentId() == null) {
+ CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(loadModel, overwriteSet);
+ }
+ // Take segment lock
+ segmentLock = CarbonLockFactory.getCarbonLockObj(
+ loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier(),
+ CarbonTablePath.addSegmentPrefix(loadModel.getSegmentId()) + LockUsage.LOCK);
+ if (!segmentLock.lockWithRetries()) {
+ throw new RuntimeException("Already segment is locked for loading, not supposed happen");
+ }
CarbonTableOutputFormat.setLoadModel(context.getConfiguration(), loadModel);
}
@@ -93,42 +113,59 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
boolean overwriteSet = CarbonTableOutputFormat.isOverwriteSet(context.getConfiguration());
CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
LoadMetadataDetails newMetaEntry = loadModel.getCurrentLoadMetadataDetail();
- String segmentPath =
- CarbonTablePath.getSegmentPath(loadModel.getTablePath(), loadModel.getSegmentId());
+ String readPath = CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath())
+ + CarbonCommonConstants.FILE_SEPARATOR
+ + loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp() + ".tmp";
// Merge all partition files into a single file.
- new PartitionMapFileStore().mergePartitionMapFiles(segmentPath,
- loadModel.getFactTimeStamp() + "");
- CarbonLoaderUtil.populateNewLoadMetaEntry(newMetaEntry, SegmentStatus.SUCCESS,
- loadModel.getFactTimeStamp(), true);
+ String segmentFileName = loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp();
+ SegmentFileStore.SegmentFile segmentFile = SegmentFileStore
+ .mergeSegmentFiles(readPath, segmentFileName,
+ CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath()));
+ if (segmentFile != null) {
+ // Move all files from temp directory of each segment to partition directory
+ SegmentFileStore.moveFromTempFolder(segmentFile,
+ loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp() + ".tmp",
+ loadModel.getTablePath());
+ newMetaEntry.setSegmentFile(segmentFileName + CarbonTablePath.SEGMENT_EXT);
+ }
+ CarbonLoaderUtil
+ .populateNewLoadMetaEntry(newMetaEntry, SegmentStatus.SUCCESS, loadModel.getFactTimeStamp(),
+ true);
CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
long segmentSize = CarbonLoaderUtil
.addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(), carbonTable);
if (segmentSize > 0 || overwriteSet) {
Object operationContext = getOperationContext();
if (operationContext != null) {
+ ((OperationContext) operationContext)
+ .setProperty("current.segmentfile", newMetaEntry.getSegmentFile());
LoadEvents.LoadTablePreStatusUpdateEvent event =
new LoadEvents.LoadTablePreStatusUpdateEvent(carbonTable.getCarbonTableIdentifier(),
loadModel);
- LoadEvents.LoadTablePostStatusUpdateEvent postStatusUpdateEvent =
- new LoadEvents.LoadTablePostStatusUpdateEvent(loadModel);
try {
OperationListenerBus.getInstance().fireEvent(event, (OperationContext) operationContext);
- OperationListenerBus.getInstance().fireEvent(postStatusUpdateEvent,
- (OperationContext) operationContext);
} catch (Exception e) {
throw new IOException(e);
}
}
String uniqueId = null;
if (overwriteSet) {
- uniqueId = overwritePartitions(loadModel);
+ if (segmentSize == 0) {
+ newMetaEntry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
+ }
+ uniqueId = overwritePartitions(loadModel, newMetaEntry);
+ } else {
+ CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, false);
}
- CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, false);
if (operationContext != null) {
+ LoadEvents.LoadTablePostStatusUpdateEvent postStatusUpdateEvent =
+ new LoadEvents.LoadTablePostStatusUpdateEvent(loadModel);
LoadEvents.LoadTableMergePartitionEvent loadTableMergePartitionEvent =
- new LoadEvents.LoadTableMergePartitionEvent(segmentPath);
+ new LoadEvents.LoadTableMergePartitionEvent(readPath);
try {
OperationListenerBus.getInstance()
+ .fireEvent(postStatusUpdateEvent, (OperationContext) operationContext);
+ OperationListenerBus.getInstance()
.fireEvent(loadTableMergePartitionEvent, (OperationContext) operationContext);
} catch (Exception e) {
throw new IOException(e);
@@ -138,29 +175,23 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
context.getConfiguration().get(CarbonTableOutputFormat.UPADTE_TIMESTAMP, null);
String segmentsToBeDeleted =
context.getConfiguration().get(CarbonTableOutputFormat.SEGMENTS_TO_BE_DELETED, "");
- List<String> segmentDeleteList = Arrays.asList(segmentsToBeDeleted.split(","));
- Set<String> segmentSet = new HashSet<>(
+ List<Segment> segmentDeleteList = Segment.toSegmentList(segmentsToBeDeleted.split(","));
+ Set<Segment> segmentSet = new HashSet<>(
new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier())
.getValidAndInvalidSegments().getValidSegments());
if (updateTime != null) {
- CarbonUpdateUtil.updateTableMetadataStatus(
- segmentSet,
- carbonTable,
- updateTime,
- true,
+ CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, updateTime, true,
segmentDeleteList);
} else if (uniqueId != null) {
- // Update the loadstatus with update time to clear cache from driver.
- CarbonUpdateUtil.updateTableMetadataStatus(
- segmentSet,
- carbonTable,
- uniqueId,
- true,
- new ArrayList<String>());
+ CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, uniqueId, true,
+ segmentDeleteList);
}
} else {
CarbonLoaderUtil.updateTableStatusForFailure(loadModel);
}
+ if (segmentLock != null) {
+ segmentLock.unlock();
+ }
}
/**
@@ -171,41 +202,31 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
* @return
* @throws IOException
*/
- private String overwritePartitions(CarbonLoadModel loadModel) throws IOException {
+ private String overwritePartitions(CarbonLoadModel loadModel, LoadMetadataDetails newMetaEntry)
+ throws IOException {
CarbonTable table = loadModel.getCarbonDataLoadSchema().getCarbonTable();
- String currentSegmentPath =
- CarbonTablePath.getSegmentPath(loadModel.getTablePath(), loadModel.getSegmentId());
- PartitionMapFileStore partitionMapFileStore = new PartitionMapFileStore();
- partitionMapFileStore.readAllPartitionsOfSegment(currentSegmentPath);
- List<List<String>> partitionsToDrop =
- new ArrayList<List<String>>(partitionMapFileStore.getPartitionMap().values());
- if (partitionsToDrop.size() > 0) {
- List<String> validSegments =
+ SegmentFileStore fileStore = new SegmentFileStore(loadModel.getTablePath(),
+ loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp()
+ + CarbonTablePath.SEGMENT_EXT);
+ List<PartitionSpec> partitionSpecs = fileStore.getPartitionSpecs();
+
+ if (partitionSpecs != null && partitionSpecs.size() > 0) {
+ List<Segment> validSegments =
new SegmentStatusManager(table.getAbsoluteTableIdentifier()).getValidAndInvalidSegments()
.getValidSegments();
String uniqueId = String.valueOf(System.currentTimeMillis());
- try {
- // First drop the partitions from partition mapper files of each segment
- for (String segment : validSegments) {
- new PartitionMapFileStore()
- .dropPartitions(CarbonTablePath.getSegmentPath(table.getTablePath(), segment),
- new ArrayList<List<String>>(partitionsToDrop), uniqueId, false);
+ List<String> tobeUpdatedSegs = new ArrayList<>();
+ List<String> tobeDeletedSegs = new ArrayList<>();
+ // First drop the partitions from partition mapper files of each segment
+ for (Segment segment : validSegments) {
+ new SegmentFileStore(table.getTablePath(), segment.getSegmentFileName())
+ .dropPartitions(segment, partitionSpecs, uniqueId, tobeDeletedSegs, tobeUpdatedSegs);
- }
- } catch (Exception e) {
- // roll back the drop partitions from carbon store
- for (String segment : validSegments) {
- new PartitionMapFileStore()
- .commitPartitions(CarbonTablePath.getSegmentPath(table.getTablePath(), segment),
- uniqueId, false, table.getTablePath(), partitionsToDrop.get(0));
- }
}
+ newMetaEntry.setUpdateStatusFileName(uniqueId);
// Commit the removed partitions in carbon store.
- for (String segment : validSegments) {
- new PartitionMapFileStore()
- .commitPartitions(CarbonTablePath.getSegmentPath(table.getTablePath(), segment),
- uniqueId, true, table.getTablePath(), partitionsToDrop.get(0));
- }
+ CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, false, "",
+ Segment.toSegmentList(tobeDeletedSegs), Segment.toSegmentList(tobeUpdatedSegs));
return uniqueId;
}
return null;
@@ -221,17 +242,57 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
}
/**
- * Update the tablestatus as fail if any fail happens.
+ * Update the tablestatus as fail if any fail happens.And also clean up the temp folders if any
+ * are existed.
*
* @param context
* @param state
* @throws IOException
*/
@Override public void abortJob(JobContext context, JobStatus.State state) throws IOException {
- super.abortJob(context, state);
- CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
- CarbonLoaderUtil.updateTableStatusForFailure(loadModel);
- LOGGER.error("Loading failed with job status : " + state);
+ try {
+ super.abortJob(context, state);
+ CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
+ CarbonLoaderUtil.updateTableStatusForFailure(loadModel);
+ String segmentFileName = loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp();
+ LoadMetadataDetails metadataDetail = loadModel.getCurrentLoadMetadataDetail();
+ if (metadataDetail != null) {
+ // In case the segment file is already created for this job then just link it so that it
+ // will be used while cleaning.
+ if (!metadataDetail.getSegmentStatus().equals(SegmentStatus.SUCCESS)) {
+ String readPath = CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath())
+ + CarbonCommonConstants.FILE_SEPARATOR + segmentFileName
+ + CarbonTablePath.SEGMENT_EXT;
+ if (FileFactory.getCarbonFile(readPath).exists()) {
+ metadataDetail.setSegmentFile(segmentFileName + CarbonTablePath.SEGMENT_EXT);
+ }
+ }
+ }
+ // Clean the temp files
+ CarbonFile segTmpFolder = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath())
+ + CarbonCommonConstants.FILE_SEPARATOR + segmentFileName + ".tmp");
+ // delete temp segment folder
+ if (segTmpFolder.exists()) {
+ FileFactory.deleteAllCarbonFilesOfDir(segTmpFolder);
+ }
+ CarbonFile segmentFilePath = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath())
+ + CarbonCommonConstants.FILE_SEPARATOR + segmentFileName
+ + CarbonTablePath.SEGMENT_EXT);
+ // Delete the temp data folders of this job if exists
+ if (segmentFilePath.exists()) {
+ SegmentFileStore fileStore = new SegmentFileStore(loadModel.getTablePath(),
+ segmentFileName + CarbonTablePath.SEGMENT_EXT);
+ SegmentFileStore.removeTempFolder(fileStore.getLocationMap(), segmentFileName + ".tmp",
+ loadModel.getTablePath());
+ }
+ LOGGER.error("Loading failed with job status : " + state);
+ } finally {
+ if (segmentLock != null) {
+ segmentLock.unlock();
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index a1887f0..96b0b21 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -30,15 +30,16 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -256,7 +257,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
/**
* Set list of segments to access
*/
- public static void setSegmentsToAccess(Configuration configuration, List<String> validSegments) {
+ public static void setSegmentsToAccess(Configuration configuration, List<Segment> validSegments) {
configuration.set(INPUT_SEGMENT_NUMBERS, CarbonUtil.convertToString(validSegments));
}
@@ -270,7 +271,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
.getProperty(CarbonCommonConstants.CARBON_INPUT_SEGMENTS + dbName + "." + tbName, "*");
if (!segmentNumbersFromProperty.trim().equals("*")) {
CarbonTableInputFormat
- .setSegmentsToAccess(conf, Arrays.asList(segmentNumbersFromProperty.split(",")));
+ .setSegmentsToAccess(conf, Segment.toSegmentList(segmentNumbersFromProperty.split(",")));
}
}
@@ -292,25 +293,28 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
/**
* set list of partitions to prune
*/
- public static void setPartitionsToPrune(Configuration configuration, List<String> partitions) {
+ public static void setPartitionsToPrune(Configuration configuration,
+ List<PartitionSpec> partitions) {
if (partitions == null) {
return;
}
try {
- String partitionString = ObjectSerializationUtil.convertObjectToString(partitions);
+ String partitionString =
+ ObjectSerializationUtil.convertObjectToString(new ArrayList<>(partitions));
configuration.set(PARTITIONS_TO_PRUNE, partitionString);
} catch (Exception e) {
- throw new RuntimeException("Error while setting patition information to Job", e);
+ throw new RuntimeException("Error while setting patition information to Job" + partitions, e);
}
}
/**
* get list of partitions to prune
*/
- public static List<String> getPartitionsToPrune(Configuration configuration) throws IOException {
+ public static List<PartitionSpec> getPartitionsToPrune(Configuration configuration)
+ throws IOException {
String partitionString = configuration.get(PARTITIONS_TO_PRUNE);
if (partitionString != null) {
- return (List<String>) ObjectSerializationUtil.convertStringToObject(partitionString);
+ return (List<PartitionSpec>) ObjectSerializationUtil.convertStringToObject(partitionString);
}
return null;
}
@@ -345,22 +349,24 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
TableDataMap blockletMap =
DataMapStoreManager.getInstance().getDataMap(identifier, BlockletDataMap.NAME,
BlockletDataMapFactory.class.getName());
- List<String> invalidSegments = new ArrayList<>();
+ List<Segment> invalidSegments = new ArrayList<>();
List<UpdateVO> invalidTimestampsList = new ArrayList<>();
- List<String> streamSegments = null;
+ List<Segment> streamSegments = null;
+ // get all valid segments and set them into the configuration
+ SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
+ SegmentStatusManager.ValidAndInvalidSegmentsInfo segments =
+ segmentStatusManager.getValidAndInvalidSegments();
if (getValidateSegmentsToAccess(job.getConfiguration())) {
- // get all valid segments and set them into the configuration
- SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
- SegmentStatusManager.ValidAndInvalidSegmentsInfo segments =
- segmentStatusManager.getValidAndInvalidSegments();
- List<String> validSegments = segments.getValidSegments();
+ List<Segment> validSegments = segments.getValidSegments();
streamSegments = segments.getStreamSegments();
if (validSegments.size() == 0) {
return getSplitsOfStreaming(job, identifier, streamSegments);
}
- List<String> filteredSegmentToAccess = getFilteredSegment(job, validSegments);
+
+
+ List<Segment> filteredSegmentToAccess = getFilteredSegment(job, segments.getValidSegments());
if (filteredSegmentToAccess.size() == 0) {
return new ArrayList<>(0);
} else {
@@ -368,31 +374,36 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
}
// remove entry in the segment index if there are invalid segments
invalidSegments.addAll(segments.getInvalidSegments());
- for (String invalidSegmentId : invalidSegments) {
- invalidTimestampsList.add(updateStatusManager.getInvalidTimestampRange(invalidSegmentId));
+ for (Segment invalidSegmentId : invalidSegments) {
+ invalidTimestampsList
+ .add(updateStatusManager.getInvalidTimestampRange(invalidSegmentId.getSegmentNo()));
}
if (invalidSegments.size() > 0) {
blockletMap.clear(invalidSegments);
}
}
-
+ ArrayList<Segment> validAndInProgressSegments = new ArrayList<>(segments.getValidSegments());
+ // Add in progress segments also to filter it as in case of aggregate table load it loads
+ // data from in progress table.
+ validAndInProgressSegments.addAll(segments.getListOfInProgressSegments());
// get updated filtered list
- List<String> filteredSegmentToAccess = Arrays.asList(getSegmentsToAccess(job));
+ List<Segment> filteredSegmentToAccess =
+ getFilteredSegment(job, new ArrayList<>(validAndInProgressSegments));
// Clean the updated segments from memory if the update happens on segments
- List<String> toBeCleanedSegments = new ArrayList<>();
+ List<Segment> toBeCleanedSegments = new ArrayList<>();
for (SegmentUpdateDetails segmentUpdateDetail : updateStatusManager
.getUpdateStatusDetails()) {
boolean refreshNeeded =
DataMapStoreManager.getInstance().getTableSegmentRefresher(identifier)
.isRefreshNeeded(segmentUpdateDetail.getSegmentName(), updateStatusManager);
if (refreshNeeded) {
- toBeCleanedSegments.add(segmentUpdateDetail.getSegmentName());
+ toBeCleanedSegments.add(new Segment(segmentUpdateDetail.getSegmentName(), null));
}
}
// Clean segments if refresh is needed
- for (String segment : filteredSegmentToAccess) {
+ for (Segment segment : filteredSegmentToAccess) {
if (DataMapStoreManager.getInstance().getTableSegmentRefresher(identifier)
- .isRefreshNeeded(segment)) {
+ .isRefreshNeeded(segment.getSegmentNo())) {
toBeCleanedSegments.add(segment);
}
}
@@ -446,20 +457,28 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
* Return segment list after filtering out valid segments and segments set by user by
* `INPUT_SEGMENT_NUMBERS` in job configuration
*/
- private List<String> getFilteredSegment(JobContext job, List<String> validSegments) {
- String[] segmentsToAccess = getSegmentsToAccess(job);
- Set<String> segmentToAccessSet = new HashSet<>(Arrays.asList(segmentsToAccess));
- List<String> filteredSegmentToAccess = new ArrayList<>();
- if (segmentsToAccess.length == 0 || segmentsToAccess[0].equalsIgnoreCase("*")) {
+ private List<Segment> getFilteredSegment(JobContext job, List<Segment> validSegments) {
+ Segment[] segmentsToAccess = getSegmentsToAccess(job);
+ List<Segment> segmentToAccessSet =
+ new ArrayList<>(new HashSet<>(Arrays.asList(segmentsToAccess)));
+ List<Segment> filteredSegmentToAccess = new ArrayList<>();
+ if (segmentsToAccess.length == 0 || segmentsToAccess[0].getSegmentNo().equalsIgnoreCase("*")) {
filteredSegmentToAccess.addAll(validSegments);
} else {
- for (String validSegment : validSegments) {
- if (segmentToAccessSet.contains(validSegment)) {
- filteredSegmentToAccess.add(validSegment);
+ for (Segment validSegment : validSegments) {
+ int index = segmentToAccessSet.indexOf(validSegment);
+ if (index > -1) {
+ // In case of in progress reading segment, segment file name is set to the property itself
+ if (segmentToAccessSet.get(index).getSegmentFileName() != null
+ && validSegment.getSegmentFileName() == null) {
+ filteredSegmentToAccess.add(segmentToAccessSet.get(index));
+ } else {
+ filteredSegmentToAccess.add(validSegment);
+ }
}
}
if (!filteredSegmentToAccess.containsAll(segmentToAccessSet)) {
- List<String> filteredSegmentToAccessTemp = new ArrayList<>(filteredSegmentToAccess);
+ List<Segment> filteredSegmentToAccessTemp = new ArrayList<>(filteredSegmentToAccess);
filteredSegmentToAccessTemp.removeAll(segmentToAccessSet);
LOG.info(
"Segments ignored are : " + Arrays.toString(filteredSegmentToAccessTemp.toArray()));
@@ -472,15 +491,15 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
* use file list in .carbonindex file to get the split of streaming.
*/
public List<InputSplit> getSplitsOfStreaming(JobContext job, AbsoluteTableIdentifier identifier,
- List<String> streamSegments) throws IOException {
+ List<Segment> streamSegments) throws IOException {
List<InputSplit> splits = new ArrayList<InputSplit>();
if (streamSegments != null && !streamSegments.isEmpty()) {
CarbonTablePath tablePath = CarbonStorePath.getCarbonTablePath(identifier);
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
- for (String segmentId : streamSegments) {
- String segmentDir = tablePath.getSegmentDir("0", segmentId);
+ for (Segment segment : streamSegments) {
+ String segmentDir = tablePath.getSegmentDir("0", segment.getSegmentNo());
FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
if (FileFactory.isFileExist(segmentDir, fileType)) {
String indexName = CarbonTablePath.getCarbonStreamIndexFileName();
@@ -508,20 +527,20 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
long bytesRemaining = length;
while (((double) bytesRemaining) / splitSize > 1.1) {
int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
- splits.add(makeSplit(segmentId, path, length - bytesRemaining, splitSize,
- blkLocations[blkIndex].getHosts(),
+ splits.add(makeSplit(segment.getSegmentNo(), path, length - bytesRemaining,
+ splitSize, blkLocations[blkIndex].getHosts(),
blkLocations[blkIndex].getCachedHosts(), FileFormat.ROW_V1));
bytesRemaining -= splitSize;
}
if (bytesRemaining != 0) {
int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
- splits.add(makeSplit(segmentId, path, length - bytesRemaining, bytesRemaining,
- blkLocations[blkIndex].getHosts(),
+ splits.add(makeSplit(segment.getSegmentNo(), path, length - bytesRemaining,
+ bytesRemaining, blkLocations[blkIndex].getHosts(),
blkLocations[blkIndex].getCachedHosts(), FileFormat.ROW_V1));
}
} else {
//Create empty hosts array for zero length files
- splits.add(makeSplit(segmentId, path, 0, length, new String[0],
+ splits.add(makeSplit(segment.getSegmentNo(), path, 0, length, new String[0],
FileFormat.ROW_V1));
}
}
@@ -558,11 +577,11 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
List<Integer> oldPartitionIdList, PartitionInfo partitionInfo)
throws IOException {
AbsoluteTableIdentifier identifier = getAbsoluteTableIdentifier(job.getConfiguration());
- List<String> invalidSegments = new ArrayList<>();
+ List<Segment> invalidSegments = new ArrayList<>();
List<UpdateVO> invalidTimestampsList = new ArrayList<>();
- List<String> segmentList = new ArrayList<>();
- segmentList.add(targetSegment);
+ List<Segment> segmentList = new ArrayList<>();
+ segmentList.add(new Segment(targetSegment, null));
setSegmentsToAccess(job.getConfiguration(), segmentList);
try {
@@ -647,7 +666,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
* @throws IOException
*/
private List<InputSplit> getSplits(JobContext job, FilterResolverIntf filterResolver,
- List<String> validSegments, BitSet matchedPartitions, PartitionInfo partitionInfo,
+ List<Segment> validSegments, BitSet matchedPartitions, PartitionInfo partitionInfo,
List<Integer> oldPartitionIdList) throws IOException {
List<InputSplit> result = new LinkedList<InputSplit>();
@@ -683,8 +702,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
}
// When iud is done then only get delete delta files for a block
try {
- deleteDeltaFilePath =
- updateStatusManager.getDeleteDeltaFilePath(inputSplit.getPath().toString());
+ deleteDeltaFilePath = updateStatusManager
+ .getDeleteDeltaFilePath(inputSplit.getPath().toString(), inputSplit.getSegmentId());
} catch (Exception e) {
throw new IOException(e);
}
@@ -713,7 +732,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
*/
private List<org.apache.carbondata.hadoop.CarbonInputSplit> getDataBlocksOfSegment(JobContext job,
AbsoluteTableIdentifier absoluteTableIdentifier, FilterResolverIntf resolver,
- BitSet matchedPartitions, List<String> segmentIds, PartitionInfo partitionInfo,
+ BitSet matchedPartitions, List<Segment> segmentIds, PartitionInfo partitionInfo,
List<Integer> oldPartitionIdList) throws IOException {
QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
@@ -727,7 +746,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
.getDataMap(absoluteTableIdentifier, BlockletDataMap.NAME,
BlockletDataMapFactory.class.getName());
DataMapJob dataMapJob = getDataMapJob(job.getConfiguration());
- List<String> partitionsToPrune = getPartitionsToPrune(job.getConfiguration());
+ List<PartitionSpec> partitionsToPrune = getPartitionsToPrune(job.getConfiguration());
List<ExtendedBlocklet> prunedBlocklets;
if (dataMapJob != null) {
DistributableDataMapFormat datamapDstr =
@@ -891,12 +910,13 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
/**
* return valid segment to access
*/
- public String[] getSegmentsToAccess(JobContext job) {
+ private Segment[] getSegmentsToAccess(JobContext job) {
String segmentString = job.getConfiguration().get(INPUT_SEGMENT_NUMBERS, "");
if (segmentString.trim().isEmpty()) {
- return new String[0];
+ return new Segment[0];
}
- return segmentString.split(",");
+ List<Segment> segments = Segment.toSegmentList(segmentString.split(","));
+ return segments.toArray(new Segment[segments.size()]);
}
/**
@@ -907,7 +927,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
* @throws IOException
*/
public BlockMappingVO getBlockRowCount(Job job, AbsoluteTableIdentifier identifier,
- List<String> partitions) throws IOException {
+ List<PartitionSpec> partitions) throws IOException {
TableDataMap blockletMap = DataMapStoreManager.getInstance()
.getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName());
SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier);
@@ -917,7 +937,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
Map<String, Long> segmentAndBlockCountMapping = new HashMap<>();
// TODO: currently only batch segment is supported, add support for streaming table
- List<String> filteredSegment = getFilteredSegment(job, allSegments.getValidSegments());
+ List<Segment> filteredSegment = getFilteredSegment(job, allSegments.getValidSegments());
List<ExtendedBlocklet> blocklets = blockletMap.prune(filteredSegment, null, partitions);
for (ExtendedBlocklet blocklet : blocklets) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index 440720e..f93f849 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -234,6 +234,8 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Obje
loadModel.setTaskNo(taskAttemptContext.getConfiguration().get(
"carbon.outputformat.taskno",
String.valueOf(System.nanoTime())));
+ loadModel.setDataWritePath(
+ taskAttemptContext.getConfiguration().get("carbon.outputformat.writepath"));
final String[] tempStoreLocations = getTempStoreLocations(taskAttemptContext);
final CarbonOutputIteratorWrapper iteratorWrapper = new CarbonOutputIteratorWrapper();
final DataLoadExecutor dataLoadExecutor = new DataLoadExecutor();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
index 96eec6f..7c1808f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
@@ -24,8 +24,10 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
@@ -49,14 +51,14 @@ public class DistributableDataMapFormat extends FileInputFormat<Void, ExtendedBl
private String dataMapName;
- private List<String> validSegments;
+ private List<Segment> validSegments;
private String className;
- private List<String> partitions;
+ private List<PartitionSpec> partitions;
- public DistributableDataMapFormat(AbsoluteTableIdentifier identifier,
- String dataMapName, List<String> validSegments, List<String> partitions, String className) {
+ public DistributableDataMapFormat(AbsoluteTableIdentifier identifier, String dataMapName,
+ List<Segment> validSegments, List<PartitionSpec> partitions, String className) {
this.identifier = identifier;
this.dataMapName = dataMapName;
this.validSegments = validSegments;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index 86f974c..31a08fc 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -87,12 +87,12 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
absoluteTableIdentifier)
// merged segment should not be there
- val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
+ val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.map(_.getSegmentNo).toList
assert(segments.contains("0.1"))
assert(segments.contains("2.1"))
assert(!segments.contains("2"))
assert(!segments.contains("3"))
- val cacheClient = new CacheClient();
+ val cacheClient = new CacheClient()
val segmentIdentifier = new TableSegmentUniqueIdentifier(absoluteTableIdentifier, "2")
val wrapper: SegmentTaskIndexWrapper = cacheClient.getSegmentAccessClient.
getIfPresent(segmentIdentifier)
@@ -181,7 +181,7 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
absoluteTableIdentifier)
// merged segment should not be there
- val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
+ val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.map(_.getSegmentNo).toList
assert(!segments.contains("0.1"))
assert(segments.contains("0.2"))
assert(!segments.contains("2"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
index bb1f829..329161f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
@@ -123,7 +123,7 @@ class MajorCompactionStopsAfterCompaction extends QueryTest with BeforeAndAfterA
absoluteTableIdentifier)
// merged segment should not be there
- val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList
+ val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.map(_.getSegmentNo).toList
assert(segments.contains("0.1"))
assert(!segments.contains("0.2"))
assert(!segments.contains("0"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index 74216ac..c3f5d0a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -27,7 +27,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.dev.{DataMap, DataMapFactory, DataMapWriter}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
import org.apache.carbondata.core.datastore.page.ColumnPage
import org.apache.carbondata.core.indexstore.schema.FilterType
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
@@ -42,15 +42,15 @@ class C2DataMapFactory() extends DataMapFactory {
override def fireEvent(event: Event): Unit = ???
- override def clear(segmentId: String): Unit = {}
+ override def clear(segmentId: Segment): Unit = {}
override def clear(): Unit = {}
override def getDataMaps(distributable: DataMapDistributable): java.util.List[DataMap] = ???
- override def getDataMaps(segmentId: String): util.List[DataMap] = ???
+ override def getDataMaps(segmentId: Segment): util.List[DataMap] = ???
- override def createWriter(segmentId: String): DataMapWriter = DataMapWriterSuite.dataMapWriterC2Mock
+ override def createWriter(segmentId: Segment): DataMapWriter = DataMapWriterSuite.dataMapWriterC2Mock
override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, FilterType.EQUALTO)
@@ -59,7 +59,7 @@ class C2DataMapFactory() extends DataMapFactory {
*
* @return
*/
- override def toDistributable(segmentId: String): util.List[DataMapDistributable] = {
+ override def toDistributable(segmentId: Segment): util.List[DataMapDistributable] = {
???
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
index 7067ef8..f2cdd67 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
@@ -29,7 +29,7 @@ import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.dev.{DataMap, DataMapFactory, DataMapWriter}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
import org.apache.carbondata.core.datastore.page.ColumnPage
import org.apache.carbondata.core.indexstore.schema.FilterType
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
@@ -270,15 +270,15 @@ class WaitingDataMap() extends DataMapFactory {
override def fireEvent(event: Event): Unit = ???
- override def clear(segmentId: String): Unit = {}
+ override def clear(segmentId: Segment): Unit = {}
override def clear(): Unit = {}
override def getDataMaps(distributable: DataMapDistributable): java.util.List[DataMap] = ???
- override def getDataMaps(segmentId: String): util.List[DataMap] = ???
+ override def getDataMaps(segmentId: Segment): util.List[DataMap] = ???
- override def createWriter(segmentId: String): DataMapWriter = {
+ override def createWriter(segmentId: Segment): DataMapWriter = {
new DataMapWriter {
override def onPageAdded(blockletId: Int, pageId: Int, pages: Array[ColumnPage]): Unit = { }
@@ -300,5 +300,5 @@ class WaitingDataMap() extends DataMapFactory {
override def getMeta: DataMapMeta = new DataMapMeta(List("o_country").asJava, FilterType.EQUALTO)
- override def toDistributable(segmentId: String): util.List[DataMapDistributable] = ???
+ override def toDistributable(segmentId: Segment): util.List[DataMapDistributable] = ???
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala
index 2e2c1f0..7b93766 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala
@@ -204,23 +204,6 @@ class StandardPartitionBadRecordLoggerTest extends QueryTest with BeforeAndAfter
)
}
- test("test load ddl command") {
- sql(
- """CREATE TABLE IF NOT EXISTS dataloadOptionTests(ID BigInt, date Timestamp, country
- String,
- actual_price Double, Quantity int, sold_price Decimal(19,2)) STORED BY 'carbondata'
- """)
- val csvFilePath = s"$resourcesPath/badrecords/emptyTimeStampValue.csv"
- try {
- sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE dataloadOptionTests OPTIONS"
- + "('bad_records_action'='FORCA', 'DELIMITER'= ',', 'QUOTECHAR'= '\"')");
- } catch {
- case ex: Exception =>
- assert("option BAD_RECORDS_ACTION can have only either FORCE or IGNORE or REDIRECT"
- .equals(ex.getMessage))
- }
- }
-
def drop(): Unit = {
sql("drop table IF EXISTS sales")
sql("drop table IF EXISTS serializable_values")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
index 0dbf1e4..7d0959c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
@@ -21,14 +21,12 @@ import java.util.concurrent.{Callable, ExecutorService, Executors}
import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.{BeforeAndAfterAll, Ignore}
+import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.constants.LoggerAction
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterAll {
var executorService: ExecutorService = _
@@ -48,20 +46,6 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
}
- def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int): Unit = {
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
- val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
- val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
- override def accept(file: CarbonFile): Boolean = {
- return file.getName.endsWith(".partitionmap")
- }
- })
- assert(dataFiles.length == partitions)
- }
-
test("data loading for global sort partition table for one partition column") {
sql(
"""
@@ -74,8 +58,6 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"', 'GLOBAL_SORT_PARTITIONS'='1')""")
- validateDataFiles("default_partitionone", "0", 1)
-
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionone order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
@@ -93,8 +75,6 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiontwo OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
- validateDataFiles("default_partitiontwo", "0", 1)
-
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitiontwo order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
@@ -112,7 +92,6 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
""".stripMargin)
sql(s"""insert into staticpartitionone PARTITION(empno='1') select empname,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary from originTable""")
- validateDataFiles("default_staticpartitionone", "0", 1)
}
test("single pass loading for global sort partition table for one partition column") {
@@ -127,7 +106,6 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE singlepasspartitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"', 'SINGLE_PASS'='true')""")
- validateDataFiles("default_singlepasspartitionone", "0", 1)
}
test("data loading for global sort partition table for one static partition column with load syntax") {
@@ -193,23 +171,27 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
| CREATE TABLE partitionmultiplethreeconcurrent (empno int, doj Timestamp,
| workgroupcategoryname String, deptno int, deptname String,
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
- | utilization int,salary int)
- | PARTITIONED BY (workgroupcategory int, empname String, designation String)
+ | utilization int,salary int,workgroupcategory int,designation String)
+ | PARTITIONED BY (empname String)
| STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('DICTIONARY_INCLUDE'='empname,designation,deptname', 'SORT_SCOPE'='GLOBAL_SORT')
+ | TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
""".stripMargin)
val tasks = new util.ArrayList[Callable[String]]()
- tasks.add(new QueryTask(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethreeconcurrent OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')"""))
- tasks.add(new QueryTask(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethreeconcurrent OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')"""))
- tasks.add(new QueryTask(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethreeconcurrent OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')"""))
+ var i = 0
+ val count = 5
+ while (i < count) {
+ tasks.add(new QueryTask(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethreeconcurrent partition(empname='ravi') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')"""))
+ i = i + 1
+ }
val results = executorService.invokeAll(tasks)
for (i <- 0 until tasks.size()) {
val res = results.get(i).get
assert("PASS".equals(res))
}
executorService.shutdown()
- checkAnswer(sql("select count(*) from partitionmultiplethreeconcurrent"), Seq(Row(30)))
+ checkAnswer(sql("select count(*) from partitionmultiplethreeconcurrent"), Seq(Row(10 * count)))
+ assert(sql("show segments for table partitionmultiplethreeconcurrent").count() == count)
}
class QueryTask(query: String) extends Callable[String] {
@@ -306,10 +288,9 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
- sql(s"""insert overwrite table staticpartitiondateinsert PARTITION(projectenddate='2016-06-29',doj='2010-12-29 00:00:00') select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary from originTable where projectenddate=cast('2016-06-29' as Date)""")
- // sql(s"""insert overwrite table partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert overwrite table staticpartitiondateinsert PARTITION(projectenddate='2016-06-29',doj='2010-12-29 00:00:00') select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary from originTable where projectenddate=cast('2016-06-29' as timestamp)""")
checkAnswer(sql("select * from staticpartitiondateinsert where projectenddate=cast('2016-06-29' as Date)"),
- sql("select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable where projectenddate=cast('2016-06-29' as Date)"))
+ sql("select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,cast(projectenddate as date),doj from originTable where projectenddate=cast('2016-06-29' as timestamp)"))
}
@@ -436,10 +417,10 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
test("test overwrite with timestamp partition column") {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
- sql("DROP TABLE IF EXISTS origintable")
+ sql("DROP TABLE IF EXISTS origintablenew")
sql(
"""
- | CREATE TABLE origintable
+ | CREATE TABLE origintablenew
| (id Int,
| vin String,
| logdate Timestamp,
@@ -452,7 +433,7 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
sql(
s"""
- LOAD DATA LOCAL INPATH '$resourcesPath/partition_data.csv' into table origintable
+ LOAD DATA LOCAL INPATH '$resourcesPath/partition_data.csv' into table origintablenew
""")
sql("DROP TABLE IF EXISTS partitiontable0")
@@ -504,9 +485,9 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
""".stripMargin))
sql("insert into table partitiontable0 partition(logdate='2018-02-15 00:00:00') " +
- "select id,vin,phonenumber,country,area,salary from origintable")
+ "select id,vin,phonenumber,country,area,salary from origintablenew")
sql("insert into table partitiontable0_hive partition(logdate='2018-02-15 00:00:00') " +
- "select id,vin,phonenumber,country,area,salary from origintable")
+ "select id,vin,phonenumber,country,area,salary from origintablenew")
checkAnswer(sql(
s"""
| SELECT logdate,id,vin,phonenumber,country,area,salary
@@ -530,10 +511,10 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
test("test overwrite with date partition column") {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
- sql("DROP TABLE IF EXISTS origintable")
+ sql("DROP TABLE IF EXISTS origintablenew")
sql(
"""
- | CREATE TABLE origintable
+ | CREATE TABLE origintablenew
| (id Int,
| vin String,
| logdate date,
@@ -546,7 +527,7 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
sql(
s"""
- LOAD DATA LOCAL INPATH '$resourcesPath/partition_data.csv' into table origintable
+ LOAD DATA LOCAL INPATH '$resourcesPath/partition_data.csv' into table origintablenew
""")
sql("DROP TABLE IF EXISTS partitiontable0")
@@ -598,9 +579,9 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
""".stripMargin))
sql("insert into table partitiontable0 partition(logdate='2018-02-15') " +
- "select id,vin,phonenumber,country,area,salary from origintable")
+ "select id,vin,phonenumber,country,area,salary from origintablenew")
sql("insert into table partitiontable0_hive partition(logdate='2018-02-15') " +
- "select id,vin,phonenumber,country,area,salary from origintable")
+ "select id,vin,phonenumber,country,area,salary from origintablenew")
checkAnswer(sql(
s"""
| SELECT logdate,id,vin,phonenumber,country,area,salary
@@ -621,7 +602,319 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
.addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
}
+ test("partition with date column issue") {
+ try {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, LoggerAction.FAIL.name())
+ sql("drop table if exists partdatecarb")
+ sql(
+ "create table partdatecarb(name string, age Int) partitioned by(dob date) stored by 'carbondata'")
+
+ sql("insert into partdatecarb partition(dob='2016-06-28') select 'name1',121")
+ checkAnswer(sql("select name,age,cast(dob as string) from partdatecarb"),
+ Seq(Row("name1", 121, "2016-06-28")))
+ } finally {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, LoggerAction.FORCE.name())
+ }
+ }
+
+ test("partition with time column issue") {
+ try {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, LoggerAction.FAIL.name())
+ sql("drop table if exists partdatecarb1")
+ sql(
+ "create table partdatecarb1(name string, age Int) partitioned by(dob timestamp) stored by 'carbondata'")
+
+ sql("insert into partdatecarb1 partition(dob='2016-06-28 00:00:00') select 'name1',121")
+ checkAnswer(sql("select name,age,cast(dob as string) from partdatecarb1"),
+ Seq(Row("name1", 121, "2016-06-28 00:00:00")))
+ } finally {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, LoggerAction.FORCE.name())
+ }
+ }
+
+ test("partition with int issue and dictionary exclude") {
+ try {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, LoggerAction.FAIL.name())
+ sql("drop table if exists partdatecarb2")
+ sql(
+ "create table partdatecarb2(name string, dob string) partitioned by(age Int) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_EXCLUDE'='age')")
+
+ sql("insert into partdatecarb2 partition(age='12') select 'name1','2016-06-28'")
+ checkAnswer(sql("select name,age,cast(dob as string) from partdatecarb2"),
+ Seq(Row("name1", 12, "2016-06-28")))
+ } finally {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, LoggerAction.FORCE.name())
+ }
+ }
+ test("partition with int issue and dictionary include") {
+ sql("drop table if exists partdatecarb3")
+ intercept[Exception] {
+ sql(
+ "create table partdatecarb3(name string, dob string) partitioned by(age Int) stored by 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='age')")
+ }
+ }
+
+ test("data loading for all dimensions with table for two partition column") {
+ sql("drop table if exists partitiontwoalldims")
+ sql(
+ """
+ | CREATE TABLE partitiontwoalldims (empno String, designation String,
+ | workgroupcategory String, workgroupcategoryname String, deptno String, deptname String,
+ | projectcode String, projectjoindate Timestamp, projectenddate Timestamp,attendance String,
+ | utilization String,salary String)
+ | PARTITIONED BY (doj Timestamp, empname String)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiontwoalldims OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ checkAnswer(sql("select count(*) from partitiontwoalldims"), Seq(Row(10)))
+ }
+
+ test("partition with different order column issue") {
+ try {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, LoggerAction.FAIL.name())
+
+ sql("drop table if exists partdatecarb4_hive")
+ sql(
+ "create table partdatecarb4_hive(name string, age Int) partitioned by(country string, state string, city string)")
+
+ sql("insert into partdatecarb4_hive partition(state,city,country='india') select 'name1',12,'KA', 'BGLR'")
+ sql("insert into partdatecarb4_hive partition(state,city,country='india') select 'name1',12,'KA', 'BGLR'")
+
+ sql("drop table if exists partdatecarb4")
+ sql(
+ "create table partdatecarb4(name string, age Int) partitioned by(country string, state string, city string) stored by 'carbondata'")
+
+ sql("insert into partdatecarb4 partition(state,city,country='india') select 'name1',12,'KA', 'BGLR'")
+ sql("insert into partdatecarb4 partition(city,state,country='india') select 'name1',12, 'BGLR','KA'")
+ sql("select * from partdatecarb4").show()
+ checkAnswer(sql("select * from partdatecarb4"), sql("select * from partdatecarb4_hive"))
+ intercept[Exception] {
+ sql(
+ "insert into partdatecarb4 partition(state,city='3',country) select 'name1',12,'cc', 'dd'")
+ }
+ } finally {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, LoggerAction.FORCE.name())
+ }
+ }
+
+ test("data loading for decimal column partition table") {
+
+ sql(
+ """
+ | CREATE TABLE partitiondecimal (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int, doj Timestamp, empname String)
+ | PARTITIONED BY (salary decimal)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiondecimal OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitiondecimal order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+ }
+
+ test("data loading for decimal column static partition table") {
+
+ sql(
+ """
+ | CREATE TABLE partitiondecimalstatic (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int, doj Timestamp, empname String)
+ | PARTITIONED BY (salary decimal)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiondecimalstatic partition(salary='1.0') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql("select count(salary) from partitiondecimalstatic"), Seq(Row(10)))
+ }
+
+ test("query after select on partition table") {
+
+ sql(
+ """
+ | CREATE TABLE partitiondatadelete (designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int,empno int, projectenddate Timestamp,attendance int,
+ | utilization int, doj Timestamp, empname String,salary int)
+ | PARTITIONED BY (projectjoindate Timestamp)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiondatadelete OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ sql(s"delete from partitiondatadelete where projectjoindate='2012-11-14 00:00:00'")
+ checkAnswer(sql(s"select count(*) from partitiondatadelete where where projectjoindate='2012-11-14 00:00:00'"), Seq(Row(0)))
+ }
+
+ test("partition colunm test without partition column in fileheader of load command") {
+ sql("DROP TABLE IF EXISTS partitiontablewithoutpartcolumninfileheader")
+
+ sql("CREATE TABLE partitiontablewithoutpartcolumninfileheader (CUST_ID int,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) partitioned by(CUST_NAME String) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='CUST_ID,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1') ")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/data_with_all_types.csv' into table partitiontablewithoutpartcolumninfileheader partition(cust_name='ravi') OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME1,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""")
+
+ checkAnswer(sql("select count(*) from partitiontablewithoutpartcolumninfileheader"), Seq(Row(10)))
+ sql("DROP TABLE IF EXISTS partitiontablewithoutpartcolumninfileheader")
+ }
+
+ test("data loading with wrong format in static partition table") {
+ sql("DROP TABLE IF EXISTS partitionwrongformat")
+ sql(
+ """
+ | CREATE TABLE partitionwrongformat (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectenddate Timestamp,attendance int,
+ | utilization int, doj Timestamp, empname String)
+ | PARTITIONED BY (projectjoindate Timestamp, salary decimal)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ intercept[MalformedCarbonCommandException] {
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionwrongformat partition(projectjoindate='2016-12-01',salary='gg') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ }
+
+ intercept[MalformedCarbonCommandException] {
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionwrongformat partition(projectjoindate='2016',salary='1.0') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ }
+
+ }
+
+ test("data loading with default partition in static partition table") {
+ sql("DROP TABLE IF EXISTS partitiondefaultpartition")
+ sql(
+ """
+ | CREATE TABLE partitiondefaultpartition (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectenddate Timestamp,attendance int,
+ | utilization int, doj Timestamp, empname String)
+ | PARTITIONED BY (projectjoindate Timestamp, salary decimal)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiondefaultpartition partition(projectjoindate='__HIVE_DEFAULT_PARTITION__',salary='1.0') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ checkAnswer(sql("select count(salary) from partitiondefaultpartition"), Seq(Row(10)))
+ checkExistence(sql("show partitions partitiondefaultpartition"), true, "__HIVE_DEFAULT_PARTITION__")
+ }
+
+ test("data loading with default partition in static partition table with fail badrecord") {
+ sql("DROP TABLE IF EXISTS partitiondefaultpartitionfail")
+ sql(
+ """
+ | CREATE TABLE partitiondefaultpartitionfail (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectenddate Timestamp,attendance int,
+ | utilization int, doj Timestamp, empname String)
+ | PARTITIONED BY (projectjoindate Timestamp, salary decimal)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiondefaultpartitionfail partition(projectjoindate='__HIVE_DEFAULT_PARTITION__',salary='1.0') OPTIONS('bad_records_logger_enable'='true', 'bad_records_action'='fail','DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ checkAnswer(sql("select count(*) from partitiondefaultpartitionfail"), Seq(Row(10)))
+ checkExistence(sql("show partitions partitiondefaultpartitionfail"), true, "__HIVE_DEFAULT_PARTITION__")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiondefaultpartitionfail partition(projectjoindate='2016-12-01',salary='__HIVE_DEFAULT_PARTITION__') OPTIONS('bad_records_logger_enable'='true', 'bad_records_action'='fail','DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ checkAnswer(sql("select count(*) from partitiondefaultpartitionfail"), Seq(Row(20)))
+ }
+
+ test("data loading with int partition issue") {
+ sql("DROP TABLE IF EXISTS intissue")
+ sql("create table intissue(a int) partitioned by (b int) stored by 'carbondata'")
+ sql("insert into intissue values(1,1)")
+ checkAnswer(sql("select * from intissue"), Seq(Row(1,1)))
+ }
+
+ test("data loading with int partition issue with global sort") {
+ sql("DROP TABLE IF EXISTS intissuesort")
+ sql("create table intissuesort(a int) partitioned by (b int) stored by 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')")
+ sql("insert into intissuesort values(1,1)")
+ checkAnswer(sql("select * from intissuesort"), Seq(Row(1,1)))
+ }
+
+ test("data loading with decimal column fail issue") {
+ sql("DROP TABLE IF EXISTS partitiondecimalfailissue")
+ sql("CREATE TABLE IF NOT EXISTS partitiondecimalfailissue (ID Int, date Timestamp, country String, name String, phonetype String, serialname String) partitioned by (salary Decimal(17,2)) STORED BY 'org.apache.carbondata.format'")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/decimalDataWithHeader.csv' into table partitiondecimalfailissue")
+ sql(s"select * from partitiondecimalfailissue").show()
+ sql(s"insert into partitiondecimalfailissue partition(salary='13000000.7878788') select ID,date,country,name,phonetype,serialname from partitiondecimalfailissue" )
+ sql(s"select * from partitiondecimalfailissue").show(100)
+ }
+
+ test("data loading with decimalissue partition issue") {
+ sql("DROP TABLE IF EXISTS decimalissue")
+ sql("create table decimalissue(a int) partitioned by (b decimal(2,2)) stored by 'carbondata'")
+ sql("insert into decimalissue values(23,21.2)")
+ checkAnswer(sql("select * from decimalissue"), Seq(Row(23,null)))
+ }
+
+ test("data loading scalar query partition issue") {
+ sql("DROP TABLE IF EXISTS scalarissue")
+ sql("create table scalarissue(a int) partitioned by (salary double) stored by 'carbondata'")
+ sql("insert into scalarissue values(23,21.2)")
+ sql("DROP TABLE IF EXISTS scalarissue_hive")
+ sql("create table scalarissue_hive(a int,salary double) using parquet partitioned by (salary) ")
+ sql("set hive.exec.dynamic.partition.mode=nonstrict")
+ sql("insert into scalarissue_hive values(23,21.2)")
+ intercept[Exception] {
+ sql(s"select * from scalarissue_hive where salary = (select max(salary) from scalarissue_hive)").show()
+ }
+ intercept[Exception] {
+ sql(s"select * from scalarissue where salary = (select max(salary) from scalarissue)").show()
+ }
+ }
+
+ test("global sort badrecords fail on partition column message") {
+ sql("DROP TABLE IF EXISTS badrecordsPartitionfailmessage")
+ sql("create table badrecordsPartitionfailmessage(intField1 int, stringField1 string) partitioned by (intField2 int) stored by 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')")
+ val ex = intercept[Exception] {
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsPartitionfailmessage options('bad_records_action'='fail')")
+ }
+ println(ex.getMessage.startsWith("DataLoad failure: Data load failed due to bad record"))
+ }
+
+ test("multiple compaction on partition table") {
+ sql("DROP TABLE IF EXISTS comp_dt2")
+ sql("create table comp_dt2(id int,name string) partitioned by (dt date,c4 int) stored by 'carbondata'")
+ sql("insert into comp_dt2 select 1,'A','2001-01-01',1")
+ sql("insert into comp_dt2 select 2,'B','2001-01-01',1")
+ sql("insert into comp_dt2 select 3,'C','2002-01-01',2")
+ sql("insert into comp_dt2 select 4,'D','2002-01-01',null")
+ assert(sql("select * from comp_dt2").collect().length == 4)
+ sql("Alter table comp_dt2 compact 'minor'")
+ assert(sql("select * from comp_dt2").collect().length == 4)
+ sql("clean files for table comp_dt2")
+ assert(sql("select * from comp_dt2").collect().length == 4)
+ sql("insert into comp_dt2 select 5,'E','2003-01-01',3")
+ sql("insert into comp_dt2 select 6,'F','2003-01-01',3")
+ sql("insert into comp_dt2 select 7,'G','2003-01-01',4")
+ sql("insert into comp_dt2 select 8,'H','2004-01-01',''")
+ assert(sql("select * from comp_dt2").collect().length == 8)
+ sql("Alter table comp_dt2 compact 'minor'")
+ sql("clean files for table comp_dt2")
+ assert(sql("select * from comp_dt2").collect().length == 8)
+ assert(sql("select * from comp_dt2").collect().length == 8)
+ sql("insert into comp_dt2 select 9,'H','2001-01-01',1")
+ sql("insert into comp_dt2 select 10,'I','2002-01-01',null")
+ sql("insert into comp_dt2 select 11,'J','2003-01-01',4")
+ sql("insert into comp_dt2 select 12,'K','2003-01-01',5")
+ assert(sql("select * from comp_dt2").collect().length == 12)
+ sql("Alter table comp_dt2 compact 'minor'")
+ assert(sql("show segments for table comp_dt2").collect().length == 8)
+ assert(sql("select * from comp_dt2").collect().length == 12)
+ sql("clean files for table comp_dt2")
+ assert(sql("select * from comp_dt2").collect().length == 12)
+ sql("insert into comp_dt2 select 13,'L','2004-01-01', 6")
+ assert(sql("select * from comp_dt2").collect().length == 13)
+ sql("Alter table comp_dt2 compact 'major'")
+ assert(sql("select * from comp_dt2").collect().length == 13)
+ assert(sql("show segments for table comp_dt2").collect().length == 3)
+ assert(sql("select * from comp_dt2").collect().length == 13)
+ sql("clean files for table comp_dt2")
+ }
override def afterAll = {
CarbonProperties.getInstance()
@@ -632,7 +925,7 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION ,
CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT)
-// dropTable
+ dropTable
if (executorService != null && !executorService.isShutdown) {
executorService.shutdownNow()
}
@@ -680,5 +973,8 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
sql("drop table if exists noLoadTable")
sql("drop table if exists carbon_test")
sql("drop table if exists carbon_test_hive")
+ sql("drop table if exists partitiondecimal")
+ sql("drop table if exists partitiondecimalstatic")
+ sql("drop table if exists partitiondatadelete")
}
}
[07/16] carbondata git commit: [MINOR]Remove dependency of Java 1.8
Posted by gv...@apache.org.
[MINOR]Remove dependency of Java 1.8
This closes #1928
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0f210c86
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0f210c86
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0f210c86
Branch: refs/heads/branch-1.3
Commit: 0f210c86ca3ee9f0fa845cdeaef418ed9253b6f8
Parents: e363dd1
Author: Zhang Zhichao <44...@qq.com>
Authored: Sun Feb 4 12:54:24 2018 +0800
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Tue Feb 27 13:17:52 2018 +0530
----------------------------------------------------------------------
.../command/preaaggregate/CreatePreAggregateTableCommand.scala | 2 +-
.../apache/spark/sql/execution/datasources/CarbonFileFormat.scala | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0f210c86/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index c8bee62..d2acb00 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -79,7 +79,7 @@ case class CreatePreAggregateTableCommand(
)
tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, neworder.mkString(","))
tableProperties.put("sort_scope", parentTable.getTableInfo.getFactTable.
- getTableProperties.getOrDefault("sort_scope", CarbonCommonConstants
+ getTableProperties.asScala.getOrElse("sort_scope", CarbonCommonConstants
.LOAD_SORT_SCOPE_DEFAULT))
tableProperties
.put(CarbonCommonConstants.TABLE_BLOCKSIZE, parentTable.getBlockSizeInMB.toString)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0f210c86/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index 99e5732..17749c8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -272,7 +272,7 @@ private class CarbonOutputWriter(path: String,
val formattedPartitions = updatedPartitions.map {case (col, value) =>
// Only convert the static partitions to the carbon format and use it while loading data
// to carbon.
- if (staticPartition.getOrDefault(col, false)) {
+ if (staticPartition.asScala.getOrElse(col, false)) {
(col, CarbonScalaUtil.convertToCarbonFormat(value,
CarbonScalaUtil.convertCarbonToSparkDataType(
table.getColumnByName(table.getTableName, col).getDataType),
[16/16] carbondata git commit: [CARBONDATA-2187][PARTITION] Partition
restructure for new folder structure and supporting partition location
feature
Posted by gv...@apache.org.
[CARBONDATA-2187][PARTITION] Partition restructure for new folder structure and supporting partition location feature
This closes #1984
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1997ca23
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1997ca23
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1997ca23
Branch: refs/heads/branch-1.3
Commit: 1997ca235f90b5746262c9654b685b9b6bd3f16a
Parents: 758d03e
Author: ravipesala <ra...@gmail.com>
Authored: Thu Feb 15 00:31:56 2018 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Tue Feb 27 13:19:35 2018 +0530
----------------------------------------------------------------------
.../core/datamap/DataMapDistributable.java | 10 +-
.../apache/carbondata/core/datamap/Segment.java | 97 +++
.../carbondata/core/datamap/TableDataMap.java | 45 +-
.../carbondata/core/datamap/dev/DataMap.java | 3 +-
.../core/datamap/dev/DataMapFactory.java | 9 +-
.../filesystem/AbstractDFSCarbonFile.java | 27 +-
.../indexstore/BlockletDataMapIndexStore.java | 107 +--
.../core/indexstore/BlockletDetailsFetcher.java | 10 +-
.../core/indexstore/PartitionSpec.java | 92 +++
.../TableBlockIndexUniqueIdentifier.java | 87 +--
.../blockletindex/BlockletDataMap.java | 134 ++--
.../BlockletDataMapDistributable.java | 4 +-
.../blockletindex/BlockletDataMapFactory.java | 104 ++-
.../blockletindex/BlockletDataMapModel.java | 27 +-
.../blockletindex/SegmentIndexFileStore.java | 94 ++-
.../carbondata/core/locks/HdfsFileLock.java | 16 -
.../carbondata/core/locks/LocalFileLock.java | 16 +-
.../core/metadata/PartitionMapFileStore.java | 484 ------------
.../core/metadata/SegmentFileStore.java | 744 +++++++++++++++++++
.../core/mutate/CarbonUpdateUtil.java | 50 +-
.../executor/impl/AbstractQueryExecutor.java | 15 +-
.../core/statusmanager/LoadMetadataDetails.java | 18 +
.../statusmanager/SegmentStatusManager.java | 87 ++-
.../SegmentUpdateStatusManager.java | 41 +-
.../apache/carbondata/core/util/CarbonUtil.java | 95 ++-
.../core/util/DataTypeConverterImpl.java | 18 +-
.../carbondata/core/util/DataTypeUtil.java | 34 +-
.../core/util/path/CarbonTablePath.java | 33 +-
.../carbondata/core/util/CarbonUtilTest.java | 9 +-
.../carbondata/hadoop/CarbonInputSplit.java | 9 +-
.../hadoop/api/CarbonOutputCommitter.java | 191 +++--
.../hadoop/api/CarbonTableInputFormat.java | 128 ++--
.../hadoop/api/CarbonTableOutputFormat.java | 2 +
.../hadoop/api/DistributableDataMapFormat.java | 10 +-
.../MajorCompactionIgnoreInMinorTest.scala | 6 +-
.../MajorCompactionStopsAfterCompaction.scala | 2 +-
.../testsuite/datamap/DataMapWriterSuite.scala | 10 +-
.../TestInsertAndOtherCommandConcurrent.scala | 10 +-
.../StandardPartitionBadRecordLoggerTest.scala | 17 -
.../StandardPartitionGlobalSortTestCase.scala | 388 ++++++++--
.../StandardPartitionTableCleanTestCase.scala | 72 +-
...andardPartitionTableCompactionTestCase.scala | 36 +-
.../StandardPartitionTableDropTestCase.scala | 1 +
.../StandardPartitionTableLoadingTestCase.scala | 62 +-
...tandardPartitionTableOverwriteTestCase.scala | 76 +-
.../StandardPartitionTableQueryTestCase.scala | 79 +-
.../spark/util/SparkDataTypeConverterImpl.java | 12 +
.../org/apache/carbondata/api/CarbonStore.scala | 12 +-
.../spark/rdd/CarbonDropPartitionRDD.scala | 118 +--
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 52 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 3 +-
.../spark/rdd/CarbonSparkPartition.scala | 3 +-
.../carbondata/spark/rdd/PartitionDropper.scala | 2 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 207 ++++--
.../carbondata/spark/util/CommonUtil.scala | 27 +-
.../carbondata/spark/util/DataLoadingUtil.scala | 88 ++-
.../command/carbonTableSchemaCommon.scala | 17 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 49 +-
.../spark/rdd/CarbonTableCompactor.scala | 72 +-
.../org/apache/spark/sql/CarbonCountStar.scala | 4 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 3 +-
.../CarbonAlterTableCompactionCommand.scala | 4 +-
.../management/CarbonCleanFilesCommand.scala | 13 +-
.../management/CarbonLoadDataCommand.scala | 432 +++++++----
.../management/RefreshCarbonTableCommand.scala | 27 +-
.../CarbonProjectForUpdateCommand.scala | 3 +-
.../command/mutation/DeleteExecution.scala | 20 +-
.../command/mutation/HorizontalCompaction.scala | 5 +-
...arbonAlterTableAddHivePartitionCommand.scala | 117 +++
...rbonAlterTableDropHivePartitionCommand.scala | 102 +--
.../CarbonAlterTableDropPartitionCommand.scala | 10 +-
.../CarbonAlterTableSplitPartitionCommand.scala | 4 +-
.../CreatePreAggregateTableCommand.scala | 7 +-
.../preaaggregate/PreAggregateListeners.scala | 13 +-
.../table/CarbonCreateTableCommand.scala | 13 +
.../datasources/CarbonFileFormat.scala | 222 +++---
.../strategy/CarbonLateDecodeStrategy.scala | 15 +-
.../sql/execution/strategy/DDLStrategy.scala | 15 +-
.../apache/spark/sql/hive/CarbonRelation.scala | 18 +-
.../spark/sql/optimizer/CarbonFilters.scala | 59 +-
.../spark/sql/hive/CarbonSessionState.scala | 12 +-
.../spark/sql/hive/CarbonSessionState.scala | 12 +-
.../datamap/DataMapWriterListener.java | 3 +-
.../loading/CarbonDataLoadConfiguration.java | 13 +
.../loading/DataLoadProcessBuilder.java | 4 +-
.../impl/MeasureFieldConverterImpl.java | 2 +-
.../loading/model/CarbonLoadModel.java | 13 +
.../processing/merger/CarbonDataMergerUtil.java | 77 +-
.../merger/CompactionResultSortProcessor.java | 35 +-
.../merger/RowResultMergerProcessor.java | 34 +-
.../partition/spliter/RowResultProcessor.java | 5 +-
.../store/CarbonFactDataHandlerModel.java | 10 +-
.../util/CarbonDataProcessorUtil.java | 8 +-
.../processing/util/CarbonLoaderUtil.java | 39 +-
.../processing/util/DeleteLoadFolders.java | 80 +-
.../carbon/datastore/BlockIndexStoreTest.java | 6 +-
.../carbondata/streaming/StreamHandoffRDD.scala | 5 +-
97 files changed, 3741 insertions(+), 1994 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
index 50af789..edd724a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
@@ -31,7 +31,7 @@ public abstract class DataMapDistributable extends InputSplit
private String tablePath;
- private String segmentId;
+ private Segment segment;
private String dataMapName;
@@ -47,12 +47,12 @@ public abstract class DataMapDistributable extends InputSplit
this.tablePath = tablePath;
}
- public String getSegmentId() {
- return segmentId;
+ public Segment getSegment() {
+ return segment;
}
- public void setSegmentId(String segmentId) {
- this.segmentId = segmentId;
+ public void setSegment(Segment segment) {
+ this.segment = segment;
}
public String getDataMapName() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java b/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java
new file mode 100644
index 0000000..c47f16c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/Segment.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Represents one load of carbondata
+ */
+public class Segment implements Serializable {
+
+ private static final long serialVersionUID = 7044555408162234064L;
+
+ private String segmentNo;
+
+ private String segmentFileName;
+
+ public Segment(String segmentNo, String segmentFileName) {
+ this.segmentNo = segmentNo;
+ this.segmentFileName = segmentFileName;
+ }
+
+ public String getSegmentNo() {
+ return segmentNo;
+ }
+
+ public String getSegmentFileName() {
+ return segmentFileName;
+ }
+
+ public static List<Segment> toSegmentList(String[] segmentIds) {
+ List<Segment> list = new ArrayList<>(segmentIds.length);
+ for (String segmentId : segmentIds) {
+ list.add(toSegment(segmentId));
+ }
+ return list;
+ }
+
+ public static List<Segment> toSegmentList(List<String> segmentIds) {
+ List<Segment> list = new ArrayList<>(segmentIds.size());
+ for (String segmentId : segmentIds) {
+ list.add(toSegment(segmentId));
+ }
+ return list;
+ }
+
+ /**
+ * SegmentId can be combination of segmentNo and segmentFileName
+ * @param segmentId
+ * @return
+ */
+ public static Segment toSegment(String segmentId) {
+ String[] split = segmentId.split("#");
+ if (split.length > 1) {
+ return new Segment(split[0], split[1]);
+ } else if (split.length > 0) {
+ return new Segment(split[0], null);
+ }
+ return new Segment(segmentId, null);
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Segment segment = (Segment) o;
+ return Objects.equals(segmentNo, segment.segmentNo);
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(segmentNo);
+ }
+
+ @Override public String toString() {
+ if (segmentFileName != null) {
+ return segmentNo + "#" + segmentFileName;
+ } else {
+ return segmentNo;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 9c84891..6555d6c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.events.Event;
@@ -60,21 +61,21 @@ public final class TableDataMap extends OperationEventListener {
/**
* Pass the valid segments and prune the datamap using filter expression
*
- * @param segmentIds
+ * @param segments
* @param filterExp
* @return
*/
- public List<ExtendedBlocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp,
- List<String> partitions) throws IOException {
+ public List<ExtendedBlocklet> prune(List<Segment> segments, FilterResolverIntf filterExp,
+ List<PartitionSpec> partitions) throws IOException {
List<ExtendedBlocklet> blocklets = new ArrayList<>();
- for (String segmentId : segmentIds) {
+ for (Segment segment : segments) {
List<Blocklet> pruneBlocklets = new ArrayList<>();
- List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(segment);
for (DataMap dataMap : dataMaps) {
pruneBlocklets.addAll(dataMap.prune(filterExp, partitions));
}
blocklets.addAll(addSegmentId(blockletDetailsFetcher
- .getExtendedBlocklets(pruneBlocklets, segmentId), segmentId));
+ .getExtendedBlocklets(pruneBlocklets, segment), segment.getSegmentNo()));
}
return blocklets;
}
@@ -94,13 +95,13 @@ public final class TableDataMap extends OperationEventListener {
*
* @return
*/
- public List<DataMapDistributable> toDistributable(List<String> segmentIds) throws IOException {
+ public List<DataMapDistributable> toDistributable(List<Segment> segments) throws IOException {
List<DataMapDistributable> distributables = new ArrayList<>();
- for (String segmentsId : segmentIds) {
- List<DataMapDistributable> list = dataMapFactory.toDistributable(segmentsId);
+ for (Segment segment : segments) {
+ List<DataMapDistributable> list = dataMapFactory.toDistributable(segment);
for (DataMapDistributable distributable: list) {
distributable.setDataMapName(dataMapName);
- distributable.setSegmentId(segmentsId);
+ distributable.setSegment(segment);
distributable.setTablePath(identifier.getTablePath());
distributable.setDataMapFactoryClass(dataMapFactory.getClass().getName());
}
@@ -118,7 +119,7 @@ public final class TableDataMap extends OperationEventListener {
* @return
*/
public List<ExtendedBlocklet> prune(DataMapDistributable distributable,
- FilterResolverIntf filterExp, List<String> partitions) throws IOException {
+ FilterResolverIntf filterExp, List<PartitionSpec> partitions) throws IOException {
List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
List<Blocklet> blocklets = new ArrayList<>();
List<DataMap> dataMaps = dataMapFactory.getDataMaps(distributable);
@@ -127,8 +128,8 @@ public final class TableDataMap extends OperationEventListener {
}
for (Blocklet blocklet: blocklets) {
ExtendedBlocklet detailedBlocklet =
- blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegmentId());
- detailedBlocklet.setSegmentId(distributable.getSegmentId());
+ blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegment());
+ detailedBlocklet.setSegmentId(distributable.getSegment().getSegmentNo());
detailedBlocklets.add(detailedBlocklet);
}
return detailedBlocklets;
@@ -136,11 +137,11 @@ public final class TableDataMap extends OperationEventListener {
/**
* Clear only the datamaps of the segments
- * @param segmentIds
+ * @param segments
*/
- public void clear(List<String> segmentIds) {
- for (String segmentId: segmentIds) {
- dataMapFactory.clear(segmentId);
+ public void clear(List<Segment> segments) {
+ for (Segment segment: segments) {
+ dataMapFactory.clear(segment);
}
}
@@ -170,21 +171,21 @@ public final class TableDataMap extends OperationEventListener {
/**
* Method to prune the segments based on task min/max values
*
- * @param segmentIds
+ * @param segments
* @param filterExp
* @return
* @throws IOException
*/
- public List<String> pruneSegments(List<String> segmentIds, FilterResolverIntf filterExp)
+ public List<String> pruneSegments(List<Segment> segments, FilterResolverIntf filterExp)
throws IOException {
List<String> prunedSegments = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- for (String segmentId : segmentIds) {
- List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
+ for (Segment segment : segments) {
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(segment);
for (DataMap dataMap : dataMaps) {
if (dataMap.isScanRequired(filterExp)) {
// If any one task in a given segment contains the data that means the segment need to
// be scanned and we need to validate further data maps in the same segment
- prunedSegments.add(segmentId);
+ prunedSegments.add(segment.getSegmentNo());
break;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index 16be1ac..f3642d6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -50,7 +51,7 @@ public interface DataMap {
* @param filterExp
* @return
*/
- List<Blocklet> prune(FilterResolverIntf filterExp, List<String> partitions);
+ List<Blocklet> prune(FilterResolverIntf filterExp, List<PartitionSpec> partitions);
// TODO Move this method to Abstract class
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index f5a7404..40cd436 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.events.Event;
@@ -37,12 +38,12 @@ public interface DataMapFactory {
/**
* Return a new write for this datamap
*/
- DataMapWriter createWriter(String segmentId);
+ DataMapWriter createWriter(Segment segment);
/**
* Get the datamap for segmentid
*/
- List<DataMap> getDataMaps(String segmentId) throws IOException;
+ List<DataMap> getDataMaps(Segment segment) throws IOException;
/**
* Get datamaps for distributable object.
@@ -53,7 +54,7 @@ public interface DataMapFactory {
* Get all distributable objects of a segmentid
* @return
*/
- List<DataMapDistributable> toDistributable(String segmentId);
+ List<DataMapDistributable> toDistributable(Segment segment);
/**
*
@@ -64,7 +65,7 @@ public interface DataMapFactory {
/**
* Clears datamap of the segment
*/
- void clear(String segmentId);
+ void clear(Segment segment);
/**
* Clear all datamaps from memory
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index 927cef5..68eaa21 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.compress.BZip2Codec;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -412,15 +413,21 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
@Override
public boolean createNewFile(String filePath, FileFactory.FileType fileType, boolean doAs,
- final FsPermission permission) throws IOException {
+ FsPermission permission) throws IOException {
filePath = filePath.replace("\\", "/");
Path path = new Path(filePath);
FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
- boolean result = fs.createNewFile(path);
- if (null != permission) {
- fs.setPermission(path, permission);
+ if (fs.exists(path)) {
+ return false;
+ } else {
+ if (permission == null) {
+ permission = FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(fs.getConf()));
+ }
+ // Pass the permissions duringg file creation itself
+ fs.create(path, permission, false, fs.getConf().getInt("io.file.buffer.size", 4096),
+ fs.getDefaultReplication(path), fs.getDefaultBlockSize(path), null).close();
+ return true;
}
- return result;
}
@Override public boolean deleteFile(String filePath, FileFactory.FileType fileType)
@@ -453,11 +460,15 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
filePath = filePath.replace("\\", "/");
Path path = new Path(filePath);
FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
- if (fs.createNewFile(path)) {
- fs.deleteOnExit(path);
+ if (fs.exists(path)) {
+ return false;
+ } else {
+ // Pass the permissions duringg file creation itself
+ fs.create(path, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL), false,
+ fs.getConf().getInt("io.file.buffer.size", 4096), fs.getDefaultReplication(path),
+ fs.getDefaultBlockSize(path), null).close();
return true;
}
- return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index 111a7a2..c6073d5 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -28,14 +28,20 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CarbonLRUCache;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapModel;
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.PartitionMapFileStore;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
+
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
/**
* Class to handle loading, unloading,clearing,storing of the table
@@ -75,22 +81,10 @@ public class BlockletDataMapIndexStore
BlockletDataMap dataMap = (BlockletDataMap) lruCache.get(lruCacheKey);
if (dataMap == null) {
try {
- String segmentPath = CarbonTablePath.getSegmentPath(
- identifier.getAbsoluteTableIdentifier().getTablePath(),
- identifier.getSegmentId());
- Map<String, BlockMetaInfo> blockMetaInfoMap = new HashMap<>();
- CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
- CarbonFile[] carbonFiles = carbonFile.locationAwareListFiles();
SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
- indexFileStore.readAllIIndexOfSegment(carbonFiles);
- PartitionMapFileStore partitionFileStore = new PartitionMapFileStore();
- partitionFileStore.readAllPartitionsOfSegment(carbonFiles, segmentPath);
- for (CarbonFile file : carbonFiles) {
- blockMetaInfoMap
- .put(file.getAbsolutePath(), new BlockMetaInfo(file.getLocations(), file.getSize()));
- }
- dataMap =
- loadAndGetDataMap(identifier, indexFileStore, partitionFileStore, blockMetaInfoMap);
+ Map<String, BlockMetaInfo> blockMetaInfoMap =
+ getBlockMetaInfoMap(identifier, indexFileStore);
+ dataMap = loadAndGetDataMap(identifier, indexFileStore, blockMetaInfoMap);
} catch (MemoryException e) {
LOGGER.error("memory exception when loading datamap: " + e.getMessage());
throw new RuntimeException(e.getMessage(), e);
@@ -99,6 +93,47 @@ public class BlockletDataMapIndexStore
return dataMap;
}
+ private Map<String, BlockMetaInfo> getBlockMetaInfoMap(TableBlockIndexUniqueIdentifier identifier,
+ SegmentIndexFileStore indexFileStore) throws IOException {
+ if (identifier.getMergeIndexFileName() != null) {
+ CarbonFile indexMergeFile = FileFactory.getCarbonFile(
+ identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier
+ .getMergeIndexFileName());
+ if (indexMergeFile.exists()) {
+ indexFileStore.readAllIIndexOfSegment(new CarbonFile[] { indexMergeFile });
+ }
+ }
+ if (indexFileStore.getFileData(identifier.getIndexFileName()) == null) {
+ indexFileStore.readAllIIndexOfSegment(new CarbonFile[] { FileFactory.getCarbonFile(
+ identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier
+ .getIndexFileName()) });
+ }
+ DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
+ Map<String, BlockMetaInfo> blockMetaInfoMap = new HashMap<>();
+ List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo(
+ identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier
+ .getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()));
+ for (DataFileFooter footer : indexInfo) {
+ String blockPath = footer.getBlockInfo().getTableBlockInfo().getFilePath();
+ blockMetaInfoMap.put(blockPath, createBlockMetaInfo(blockPath));
+ }
+ return blockMetaInfoMap;
+ }
+
+ private BlockMetaInfo createBlockMetaInfo(String carbonDataFile) throws IOException {
+ CarbonFile carbonFile = FileFactory.getCarbonFile(carbonDataFile);
+ if (carbonFile instanceof AbstractDFSCarbonFile) {
+ RemoteIterator<LocatedFileStatus> iter =
+ ((AbstractDFSCarbonFile)carbonFile).fs.listLocatedStatus(new Path(carbonDataFile));
+ LocatedFileStatus fileStatus = iter.next();
+ String[] location = fileStatus.getBlockLocations()[0].getHosts();
+ long len = fileStatus.getLen();
+ return new BlockMetaInfo(location, len);
+ } else {
+ return new BlockMetaInfo(new String[]{"localhost"}, carbonFile.getSize());
+ }
+ }
+
@Override
public List<BlockletDataMap> getAll(
List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException {
@@ -116,34 +151,13 @@ public class BlockletDataMapIndexStore
}
}
if (missedIdentifiers.size() > 0) {
- Map<String, SegmentIndexFileStore> segmentIndexFileStoreMap = new HashMap<>();
- Map<String, PartitionMapFileStore> partitionFileStoreMap = new HashMap<>();
- Map<String, BlockMetaInfo> blockMetaInfoMap = new HashMap<>();
+ SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
for (TableBlockIndexUniqueIdentifier identifier: missedIdentifiers) {
- SegmentIndexFileStore indexFileStore =
- segmentIndexFileStoreMap.get(identifier.getSegmentId());
- PartitionMapFileStore partitionFileStore =
- partitionFileStoreMap.get(identifier.getSegmentId());
- String segmentPath = CarbonTablePath.getSegmentPath(
- identifier.getAbsoluteTableIdentifier().getTablePath(),
- identifier.getSegmentId());
- if (indexFileStore == null) {
- CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
- CarbonFile[] carbonFiles = carbonFile.locationAwareListFiles();
- indexFileStore = new SegmentIndexFileStore();
- indexFileStore.readAllIIndexOfSegment(carbonFiles);
- segmentIndexFileStoreMap.put(identifier.getSegmentId(), indexFileStore);
- partitionFileStore = new PartitionMapFileStore();
- partitionFileStore.readAllPartitionsOfSegment(carbonFiles, segmentPath);
- partitionFileStoreMap.put(identifier.getSegmentId(), partitionFileStore);
- for (CarbonFile file : carbonFiles) {
- blockMetaInfoMap.put(FileFactory.getUpdatedFilePath(file.getAbsolutePath()),
- new BlockMetaInfo(file.getLocations(), file.getSize()));
- }
- }
+ Map<String, BlockMetaInfo> blockMetaInfoMap =
+ getBlockMetaInfoMap(identifier, indexFileStore);
blockletDataMaps.add(
- loadAndGetDataMap(identifier, indexFileStore, partitionFileStore, blockMetaInfoMap));
+ loadAndGetDataMap(identifier, indexFileStore, blockMetaInfoMap));
}
}
} catch (Throwable e) {
@@ -194,7 +208,6 @@ public class BlockletDataMapIndexStore
private BlockletDataMap loadAndGetDataMap(
TableBlockIndexUniqueIdentifier identifier,
SegmentIndexFileStore indexFileStore,
- PartitionMapFileStore partitionFileStore,
Map<String, BlockMetaInfo> blockMetaInfoMap)
throws IOException, MemoryException {
String uniqueTableSegmentIdentifier =
@@ -206,10 +219,10 @@ public class BlockletDataMapIndexStore
BlockletDataMap dataMap;
synchronized (lock) {
dataMap = new BlockletDataMap();
- dataMap.init(new BlockletDataMapModel(identifier.getFilePath(),
- indexFileStore.getFileData(identifier.getCarbonIndexFileName()),
- partitionFileStore.getPartitions(identifier.getCarbonIndexFileName()),
- partitionFileStore.isPartionedSegment(), blockMetaInfoMap));
+ dataMap.init(new BlockletDataMapModel(
+ identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier
+ .getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()),
+ blockMetaInfoMap, identifier.getSegmentId()));
lruCache.put(identifier.getUniqueTableSegmentIdentifier(), dataMap,
dataMap.getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
index 21ecba1..b4d6db2 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailsFetcher.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.core.indexstore;
import java.io.IOException;
import java.util.List;
+import org.apache.carbondata.core.datamap.Segment;
+
/**
* Fetches the detailed blocklet which has more information to execute the query
*/
@@ -28,20 +30,20 @@ public interface BlockletDetailsFetcher {
* Get the blocklet detail information based on blockletid, blockid and segmentid.
*
* @param blocklets
- * @param segmentId
+ * @param segment
* @return
* @throws IOException
*/
- List<ExtendedBlocklet> getExtendedBlocklets(List<Blocklet> blocklets, String segmentId)
+ List<ExtendedBlocklet> getExtendedBlocklets(List<Blocklet> blocklets, Segment segment)
throws IOException;
/**
* Get the blocklet detail information based on blockletid, blockid and segmentid.
*
* @param blocklet
- * @param segmentId
+ * @param segment
* @return
* @throws IOException
*/
- ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, String segmentId) throws IOException;
+ ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, Segment segment) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/indexstore/PartitionSpec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/PartitionSpec.java b/core/src/main/java/org/apache/carbondata/core/indexstore/PartitionSpec.java
new file mode 100644
index 0000000..87c875e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/PartitionSpec.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.Serializable;
+import java.net.URI;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Holds partition information.
+ */
+public class PartitionSpec implements Serializable {
+
+ private static final long serialVersionUID = 4828007433384867678L;
+
+ /**
+ * It holds the partition information in columnName=partitionValue combination.
+ */
+ private List<String> partitions;
+
+ private transient Path locationPath;
+
+ private String location;
+
+ private String uuid;
+
+ public PartitionSpec(List<String> partitions, String location) {
+ this.partitions = partitions;
+ this.locationPath = new Path(FileFactory.getUpdatedFilePath(location));
+ this.location = locationPath.toString();
+ }
+
+ public PartitionSpec(List<String> partitions, URI location) {
+ this.partitions = partitions;
+ this.locationPath = new Path(FileFactory.getUpdatedFilePath(new Path(location).toString()));
+ this.location = locationPath.toString();
+ }
+
+ public List<String> getPartitions() {
+ return partitions;
+ }
+
+ public Path getLocation() {
+ if (locationPath == null) {
+ locationPath = new Path(location);
+ }
+ return locationPath;
+ }
+
+ public String getUuid() {
+ return uuid;
+ }
+
+ public void setUuid(String uuid) {
+ this.uuid = uuid;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ PartitionSpec spec = (PartitionSpec) o;
+ return Objects.equals(getLocation(), spec.getLocation());
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(locationPath);
+ }
+
+ @Override public String toString() {
+ return "PartitionSpec{" + "partitions=" + partitions + ", locationPath=" + locationPath
+ + ", location='" + location + '\'' + '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifier.java
index 18357ac..c907fa8 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifier.java
@@ -17,91 +17,66 @@
package org.apache.carbondata.core.indexstore;
+import java.util.Objects;
+
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
/**
- * Class holds the absoluteTableIdentifier and segmentId to uniquely identify a segment
+ * Class holds the indexFile information to uniquely identitify the carbon index
*/
public class TableBlockIndexUniqueIdentifier {
- /**
- * table fully qualified identifier
- */
- private AbsoluteTableIdentifier absoluteTableIdentifier;
- private String segmentId;
+ private String indexFilePath;
- private String carbonIndexFileName;
+ private String indexFileName;
- /**
- * Constructor to initialize the class instance
- *
- * @param absoluteTableIdentifier
- * @param segmentId
- */
- public TableBlockIndexUniqueIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier,
- String segmentId, String carbonIndexFileName) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
+ private String mergeIndexFileName;
+
+ private String segmentId;
+
+ public TableBlockIndexUniqueIdentifier(String indexFilePath, String indexFileName,
+ String mergeIndexFileName, String segmentId) {
+ this.indexFilePath = indexFilePath;
+ this.indexFileName = indexFileName;
+ this.mergeIndexFileName = mergeIndexFileName;
this.segmentId = segmentId;
- this.carbonIndexFileName = carbonIndexFileName;
}
/**
- * returns AbsoluteTableIdentifier
+ * method returns the id to uniquely identify a key
*
* @return
*/
- public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
- return absoluteTableIdentifier;
+ public String getUniqueTableSegmentIdentifier() {
+ return indexFilePath + CarbonCommonConstants.FILE_SEPARATOR + indexFileName;
}
- public String getSegmentId() {
- return segmentId;
+ public String getIndexFilePath() {
+ return indexFilePath;
}
- /**
- * method returns the id to uniquely identify a key
- *
- * @return
- */
- public String getUniqueTableSegmentIdentifier() {
- CarbonTableIdentifier carbonTableIdentifier =
- absoluteTableIdentifier.getCarbonTableIdentifier();
- return carbonTableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR
- + carbonTableIdentifier.getTableName() + CarbonCommonConstants.UNDERSCORE
- + carbonTableIdentifier.getTableId() + CarbonCommonConstants.FILE_SEPARATOR + segmentId
- + CarbonCommonConstants.FILE_SEPARATOR + carbonIndexFileName;
+ public String getIndexFileName() {
+ return indexFileName;
+ }
+
+ public String getMergeIndexFileName() {
+ return mergeIndexFileName;
}
- public String getFilePath() {
- return absoluteTableIdentifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + "/"
- + carbonIndexFileName;
+ public String getSegmentId() {
+ return segmentId;
}
@Override public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
-
TableBlockIndexUniqueIdentifier that = (TableBlockIndexUniqueIdentifier) o;
-
- if (!absoluteTableIdentifier.equals(that.absoluteTableIdentifier)) {
- return false;
- }
- if (!segmentId.equals(that.segmentId)) {
- return false;
- }
- return carbonIndexFileName.equals(that.carbonIndexFileName);
+ return Objects.equals(indexFilePath, that.indexFilePath) && Objects
+ .equals(indexFileName, that.indexFileName) && Objects
+ .equals(mergeIndexFileName, that.mergeIndexFileName);
}
@Override public int hashCode() {
- int result = absoluteTableIdentifier.hashCode();
- result = 31 * result + segmentId.hashCode();
- result = 31 * result + carbonIndexFileName.hashCode();
- return result;
- }
-
- public String getCarbonIndexFileName() {
- return carbonIndexFileName;
+ return Objects.hash(indexFilePath, indexFileName, mergeIndexFileName);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 699f9e1..9ec7a46 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.indexstore.BlockMetaInfo;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore;
import org.apache.carbondata.core.indexstore.row.DataMapRow;
import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
@@ -65,8 +66,10 @@ import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataFileFooterConverter;
import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.Path;
import org.xerial.snappy.Snappy;
/**
@@ -111,7 +114,11 @@ public class BlockletDataMap implements DataMap, Cacheable {
private static int SCHEMA = 2;
- private static int PARTITION_INFO = 3;
+ private static int INDEX_PATH = 3;
+
+ private static int INDEX_FILE_NAME = 4;
+
+ private static int SEGMENTID = 5;
private UnsafeMemoryDMStore unsafeMemoryDMStore;
@@ -121,8 +128,6 @@ public class BlockletDataMap implements DataMap, Cacheable {
private int[] columnCardinality;
- private boolean isPartitionedSegment;
-
@Override
public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
long startTime = System.currentTimeMillis();
@@ -131,7 +136,11 @@ public class BlockletDataMap implements DataMap, Cacheable {
DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
List<DataFileFooter> indexInfo = fileFooterConverter
.getIndexInfo(blockletDataMapInfo.getFilePath(), blockletDataMapInfo.getFileData());
- isPartitionedSegment = blockletDataMapInfo.isPartitionedSegment();
+ Path path = new Path(blockletDataMapInfo.getFilePath());
+ byte[] filePath = path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ byte[] fileName = path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ byte[] segmentId =
+ blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
DataMapRowImpl summaryRow = null;
byte[] schemaBinary = null;
// below 2 variables will be used for fetching the relative blocklet id. Relative blocklet ID
@@ -145,7 +154,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
createSchema(segmentProperties);
- createSummarySchema(segmentProperties, blockletDataMapInfo.getPartitions(), schemaBinary);
+ createSummarySchema(segmentProperties, schemaBinary, filePath, fileName,
+ segmentId);
}
TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
BlockMetaInfo blockMetaInfo =
@@ -182,8 +192,10 @@ public class BlockletDataMap implements DataMap, Cacheable {
if (null != unsafeMemorySummaryDMStore) {
addTaskSummaryRowToUnsafeMemoryStore(
summaryRow,
- blockletDataMapInfo.getPartitions(),
- schemaBinary);
+ schemaBinary,
+ filePath,
+ fileName,
+ segmentId);
unsafeMemorySummaryDMStore.finishWriting();
}
LOGGER.info(
@@ -354,8 +366,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
return summaryRow;
}
- private void addTaskSummaryRowToUnsafeMemoryStore(DataMapRow summaryRow,
- List<String> partitions, byte[] schemaBinary) throws IOException {
+ private void addTaskSummaryRowToUnsafeMemoryStore(DataMapRow summaryRow, byte[] schemaBinary,
+ byte[] filePath, byte[] fileName, byte[] segmentId) {
// write the task summary info to unsafe memory store
if (null != summaryRow) {
// Add column schema , it is useful to generate segment properties in executor.
@@ -363,18 +375,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
if (schemaBinary != null) {
summaryRow.setByteArray(schemaBinary, SCHEMA);
}
- if (partitions != null && partitions.size() > 0) {
- CarbonRowSchema[] minSchemas =
- ((CarbonRowSchema.StructCarbonRowSchema) unsafeMemorySummaryDMStore
- .getSchema()[PARTITION_INFO]).getChildSchemas();
- DataMapRow partitionRow = new DataMapRowImpl(minSchemas);
- for (int i = 0; i < partitions.size(); i++) {
- partitionRow
- .setByteArray(partitions.get(i).getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS),
- i);
- }
- summaryRow.setRow(partitionRow, PARTITION_INFO);
- }
+ summaryRow.setByteArray(filePath, INDEX_PATH);
+ summaryRow.setByteArray(fileName, INDEX_FILE_NAME);
+ summaryRow.setByteArray(segmentId, SEGMENTID);
try {
unsafeMemorySummaryDMStore.addIndexRowToUnsafe(summaryRow);
} catch (Exception e) {
@@ -560,27 +563,25 @@ public class BlockletDataMap implements DataMap, Cacheable {
* once per datamap. It stores datamap level max/min of each column and partition information of
* datamap
* @param segmentProperties
- * @param partitions
* @throws MemoryException
*/
- private void createSummarySchema(SegmentProperties segmentProperties, List<String> partitions,
- byte[] schemaBinary)
+ private void createSummarySchema(SegmentProperties segmentProperties, byte[] schemaBinary,
+ byte[] filePath, byte[] fileName, byte[] segmentId)
throws MemoryException {
List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>();
getMinMaxSchema(segmentProperties, taskMinMaxSchemas);
// for storing column schema
taskMinMaxSchemas.add(
new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, schemaBinary.length));
- if (partitions != null && partitions.size() > 0) {
- CarbonRowSchema[] mapSchemas = new CarbonRowSchema[partitions.size()];
- for (int i = 0; i < mapSchemas.length; i++) {
- mapSchemas[i] = new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY);
- }
- CarbonRowSchema mapSchema =
- new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),
- mapSchemas);
- taskMinMaxSchemas.add(mapSchema);
- }
+ // for storing file path
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, filePath.length));
+ // for storing file name
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, fileName.length));
+ // for storing segmentid
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, segmentId.length));
unsafeMemorySummaryDMStore = new UnsafeMemoryDMStore(
taskMinMaxSchemas.toArray(new CarbonRowSchema[taskMinMaxSchemas.size()]));
}
@@ -660,22 +661,24 @@ public class BlockletDataMap implements DataMap, Cacheable {
return blocklets;
}
- @Override public List<Blocklet> prune(FilterResolverIntf filterExp, List<String> partitions) {
+ @Override
+ public List<Blocklet> prune(FilterResolverIntf filterExp, List<PartitionSpec> partitions) {
if (unsafeMemoryDMStore.getRowCount() == 0) {
return new ArrayList<>();
}
- // First get the partitions which are stored inside datamap.
- List<String> storedPartitions = getPartitions();
// if it has partitioned datamap but there is no partitioned information stored, it means
// partitions are dropped so return empty list.
- if (isPartitionedSegment && (storedPartitions == null || storedPartitions.size() == 0)) {
- return new ArrayList<>();
- }
- if (storedPartitions != null && storedPartitions.size() > 0) {
+ if (partitions != null) {
+ // First get the partitions which are stored inside datamap.
+ String[] fileDetails = getFileDetails();
// Check the exact match of partition information inside the stored partitions.
boolean found = false;
- if (partitions != null && partitions.size() > 0) {
- found = partitions.containsAll(storedPartitions);
+ Path folderPath = new Path(fileDetails[0]);
+ for (PartitionSpec spec : partitions) {
+ if (folderPath.equals(spec.getLocation()) && isCorrectUUID(fileDetails, spec)) {
+ found = true;
+ break;
+ }
}
if (!found) {
return new ArrayList<>();
@@ -685,6 +688,20 @@ public class BlockletDataMap implements DataMap, Cacheable {
return prune(filterExp);
}
+ private boolean isCorrectUUID(String[] fileDetails, PartitionSpec spec) {
+ boolean needToScan = false;
+ if (spec.getUuid() != null) {
+ String[] split = spec.getUuid().split("_");
+ if (split[0].equals(fileDetails[2]) && CarbonTablePath.DataFileUtil
+ .getTimeStampFromFileName(fileDetails[1]).equals(split[1])) {
+ needToScan = true;
+ }
+ } else {
+ needToScan = true;
+ }
+ return needToScan;
+ }
+
/**
* select the blocks based on column min and max value
*
@@ -767,6 +784,23 @@ public class BlockletDataMap implements DataMap, Cacheable {
return blocklet;
}
+ private String[] getFileDetails() {
+ try {
+ String[] fileDetails = new String[3];
+ DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
+ fileDetails[0] =
+ new String(unsafeRow.getByteArray(INDEX_PATH), CarbonCommonConstants.DEFAULT_CHARSET);
+ fileDetails[1] = new String(unsafeRow.getByteArray(INDEX_FILE_NAME),
+ CarbonCommonConstants.DEFAULT_CHARSET);
+ fileDetails[2] = new String(unsafeRow.getByteArray(SEGMENTID),
+ CarbonCommonConstants.DEFAULT_CHARSET);
+ return fileDetails;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
/**
* Binary search used to get the first tentative index row based on
* search key
@@ -874,20 +908,6 @@ public class BlockletDataMap implements DataMap, Cacheable {
return dataMapRow;
}
- private List<String> getPartitions() {
- DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
- if (unsafeRow.getColumnCount() > PARTITION_INFO) {
- List<String> partitions = new ArrayList<>();
- DataMapRow row = unsafeRow.getRow(PARTITION_INFO);
- for (int i = 0; i < row.getColumnCount(); i++) {
- partitions.add(
- new String(row.getByteArray(i), CarbonCommonConstants.DEFAULT_CHARSET_CLASS));
- }
- return partitions;
- }
- return null;
- }
-
private byte[] getColumnSchemaBinary() {
DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
return unsafeRow.getByteArray(SCHEMA);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapDistributable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapDistributable.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapDistributable.java
index 63f45b5..99e48a5 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapDistributable.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapDistributable.java
@@ -31,8 +31,8 @@ public class BlockletDataMapDistributable extends DataMapDistributable {
*/
private String filePath;
- public BlockletDataMapDistributable(String indexFileName) {
- this.filePath = indexFileName;
+ public BlockletDataMapDistributable(String indexFilePath) {
+ this.filePath = indexFilePath;
}
public String getFilePath() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 2e2cab5..5eb077f 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datamap.dev.DataMapWriter;
@@ -37,6 +38,7 @@ import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -65,30 +67,40 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override
- public DataMapWriter createWriter(String segmentId) {
+ public DataMapWriter createWriter(Segment segment) {
throw new UnsupportedOperationException("not implemented");
}
@Override
- public List<DataMap> getDataMaps(String segmentId) throws IOException {
+ public List<DataMap> getDataMaps(Segment segment) throws IOException {
List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
- getTableBlockIndexUniqueIdentifiers(segmentId);
+ getTableBlockIndexUniqueIdentifiers(segment);
return cache.getAll(tableBlockIndexUniqueIdentifiers);
}
private List<TableBlockIndexUniqueIdentifier> getTableBlockIndexUniqueIdentifiers(
- String segmentId) throws IOException {
+ Segment segment) throws IOException {
List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
- segmentMap.get(segmentId);
+ segmentMap.get(segment.getSegmentNo());
if (tableBlockIndexUniqueIdentifiers == null) {
tableBlockIndexUniqueIdentifiers = new ArrayList<>();
- String path = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
- List<String> indexFiles = new SegmentIndexFileStore().getIndexFilesFromSegment(path);
- for (int i = 0; i < indexFiles.size(); i++) {
+ Map<String, String> indexFiles;
+ if (segment.getSegmentFileName() == null) {
+ String path =
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo());
+ indexFiles = new SegmentIndexFileStore().getIndexFilesFromSegment(path);
+ } else {
+ SegmentFileStore fileStore =
+ new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
+ indexFiles = fileStore.getIndexFiles();
+ }
+ for (Map.Entry<String, String> indexFileEntry: indexFiles.entrySet()) {
+ Path indexFile = new Path(indexFileEntry.getKey());
tableBlockIndexUniqueIdentifiers.add(
- new TableBlockIndexUniqueIdentifier(identifier, segmentId, indexFiles.get(i)));
+ new TableBlockIndexUniqueIdentifier(indexFile.getParent().toString(),
+ indexFile.getName(), indexFileEntry.getValue(), segment.getSegmentNo()));
}
- segmentMap.put(segmentId, tableBlockIndexUniqueIdentifiers);
+ segmentMap.put(segment.getSegmentNo(), tableBlockIndexUniqueIdentifiers);
}
return tableBlockIndexUniqueIdentifiers;
}
@@ -99,7 +111,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
* datamap.
*/
@Override
- public List<ExtendedBlocklet> getExtendedBlocklets(List<Blocklet> blocklets, String segmentId)
+ public List<ExtendedBlocklet> getExtendedBlocklets(List<Blocklet> blocklets, Segment segment)
throws IOException {
List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
// If it is already detailed blocklet then type cast and return same
@@ -110,7 +122,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
return detailedBlocklets;
}
List<TableBlockIndexUniqueIdentifier> identifiers =
- getTableBlockIndexUniqueIdentifiers(segmentId);
+ getTableBlockIndexUniqueIdentifiers(segment);
// Retrieve each blocklets detail information from blocklet datamap
for (Blocklet blocklet : blocklets) {
detailedBlocklets.add(getExtendedBlocklet(identifiers, blocklet));
@@ -119,13 +131,13 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
}
@Override
- public ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, String segmentId)
+ public ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, Segment segment)
throws IOException {
if (blocklet instanceof ExtendedBlocklet) {
return (ExtendedBlocklet) blocklet;
}
List<TableBlockIndexUniqueIdentifier> identifiers =
- getTableBlockIndexUniqueIdentifiers(segmentId);
+ getTableBlockIndexUniqueIdentifiers(segment);
return getExtendedBlocklet(identifiers, blocklet);
}
@@ -133,7 +145,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
Blocklet blocklet) throws IOException {
String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getPath());
for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
- if (identifier.getCarbonIndexFileName().equals(carbonIndexFileName)) {
+ if (identifier.getIndexFilePath().equals(carbonIndexFileName)) {
DataMap dataMap = cache.get(identifier);
return ((BlockletDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId());
}
@@ -141,36 +153,51 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
throw new IOException("Blocklet with blockid " + blocklet.getPath() + " not found ");
}
-
@Override
- public List<DataMapDistributable> toDistributable(String segmentId) {
- CarbonFile[] carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(segmentId);
+ public List<DataMapDistributable> toDistributable(Segment segment) {
List<DataMapDistributable> distributables = new ArrayList<>();
- for (int i = 0; i < carbonIndexFiles.length; i++) {
- Path path = new Path(carbonIndexFiles[i].getPath());
- try {
+ try {
+ CarbonFile[] carbonIndexFiles;
+ if (segment.getSegmentFileName() == null) {
+ carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo()));
+ } else {
+ SegmentFileStore fileStore =
+ new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
+ Map<String, String> indexFiles = fileStore.getIndexFiles();
+ carbonIndexFiles = new CarbonFile[indexFiles.size()];
+ int i = 0;
+ for (String indexFile : indexFiles.keySet()) {
+ carbonIndexFiles[i++] = FileFactory.getCarbonFile(indexFile);
+ }
+ }
+ for (int i = 0; i < carbonIndexFiles.length; i++) {
+ Path path = new Path(carbonIndexFiles[i].getPath());
+
FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
LocatedFileStatus fileStatus = iter.next();
String[] location = fileStatus.getBlockLocations()[0].getHosts();
BlockletDataMapDistributable distributable =
- new BlockletDataMapDistributable(path.getName());
+ new BlockletDataMapDistributable(path.toString());
distributable.setLocations(location);
distributables.add(distributable);
- } catch (IOException e) {
- throw new RuntimeException(e);
+
}
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
return distributables;
}
- @Override public void fireEvent(Event event) {
+ @Override
+ public void fireEvent(Event event) {
}
@Override
- public void clear(String segmentId) {
- List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segmentId);
+ public void clear(Segment segment) {
+ List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segment.getSegmentNo());
if (blockIndexes != null) {
for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
DataMap dataMap = cache.getIfPresent(blockIndex);
@@ -185,7 +212,7 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
@Override
public void clear() {
for (String segmentId : segmentMap.keySet().toArray(new String[segmentMap.size()])) {
- clear(segmentId);
+ clear(new Segment(segmentId, null));
}
}
@@ -193,18 +220,21 @@ public class BlockletDataMapFactory implements DataMapFactory, BlockletDetailsFe
public List<DataMap> getDataMaps(DataMapDistributable distributable) throws IOException {
BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
- if (mapDistributable.getFilePath().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
- identifiers.add(new TableBlockIndexUniqueIdentifier(identifier, distributable.getSegmentId(),
- mapDistributable.getFilePath()));
- } else if (mapDistributable.getFilePath().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
+ Path indexPath = new Path(mapDistributable.getFilePath());
+ String segmentNo = mapDistributable.getSegment().getSegmentNo();
+ if (indexPath.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
+ String parent = indexPath.getParent().toString();
+ identifiers
+ .add(new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo));
+ } else if (indexPath.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
- List<String> indexFiles = fileStore.getIndexFilesFromMergeFile(
- CarbonTablePath.getSegmentPath(identifier.getTablePath(), mapDistributable.getSegmentId())
- + "/" + mapDistributable.getFilePath());
+ CarbonFile carbonFile = FileFactory.getCarbonFile(indexPath.toString());
+ String parentPath = carbonFile.getParentFile().getAbsolutePath();
+ List<String> indexFiles = fileStore.getIndexFilesFromMergeFile(carbonFile.getAbsolutePath());
for (String indexFile : indexFiles) {
identifiers.add(
- new TableBlockIndexUniqueIdentifier(identifier, distributable.getSegmentId(),
- indexFile));
+ new TableBlockIndexUniqueIdentifier(parentPath, indexFile, carbonFile.getName(),
+ segmentNo));
}
}
List<DataMap> dataMaps;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
index b3a7f8c..ebeb278 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
@@ -16,7 +16,6 @@
*/
package org.apache.carbondata.core.indexstore.blockletindex;
-import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
@@ -29,35 +28,27 @@ public class BlockletDataMapModel extends DataMapModel {
private byte[] fileData;
- private List<String> partitions;
+ private Map<String, BlockMetaInfo> blockMetaInfoMap;
- private boolean partitionedSegment;
+ private String segmentId;
- Map<String, BlockMetaInfo> blockMetaInfoMap;
-
- public BlockletDataMapModel(String filePath, byte[] fileData, List<String> partitions,
- boolean partitionedSegment,
- Map<String, BlockMetaInfo> blockMetaInfoMap) {
+ public BlockletDataMapModel(String filePath, byte[] fileData,
+ Map<String, BlockMetaInfo> blockMetaInfoMap, String segmentId) {
super(filePath);
this.fileData = fileData;
- this.partitions = partitions;
- this.partitionedSegment = partitionedSegment;
this.blockMetaInfoMap = blockMetaInfoMap;
+ this.segmentId = segmentId;
}
public byte[] getFileData() {
return fileData;
}
- public List<String> getPartitions() {
- return partitions;
- }
-
- public boolean isPartitionedSegment() {
- return partitionedSegment;
- }
-
public Map<String, BlockMetaInfo> getBlockMetaInfoMap() {
return blockMetaInfoMap;
}
+
+ public String getSegmentId() {
+ return segmentId;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
index a30b04c..b88c1f4 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
@@ -28,10 +28,12 @@ import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.reader.ThriftReader;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataFileFooterConverter;
@@ -58,8 +60,14 @@ public class SegmentIndexFileStore {
*/
private Map<String, byte[]> carbonIndexMap;
+ /**
+ * Stores the indexfile name and related binary file data in it.
+ */
+ private Map<String, byte[]> carbonIndexMapWithFullPath;
+
public SegmentIndexFileStore() {
carbonIndexMap = new HashMap<>();
+ carbonIndexMapWithFullPath = new HashMap<>();
}
/**
@@ -80,6 +88,45 @@ public class SegmentIndexFileStore {
}
/**
+ * Read all index files and keep the cache in it.
+ *
+ * @param segmentFileStore
+ * @throws IOException
+ */
+ public void readAllIIndexOfSegment(SegmentFileStore segmentFileStore, SegmentStatus status,
+ boolean ignoreStatus) throws IOException {
+ List<CarbonFile> carbonIndexFiles = new ArrayList<>();
+ if (segmentFileStore.getLocationMap() == null) {
+ return;
+ }
+ for (Map.Entry<String, SegmentFileStore.FolderDetails> locations : segmentFileStore
+ .getLocationMap().entrySet()) {
+ String location = locations.getKey();
+
+ if (locations.getValue().getStatus().equals(status.getMessage()) || ignoreStatus) {
+ if (locations.getValue().isRelative()) {
+ location =
+ segmentFileStore.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + location;
+ }
+ for (String indexFile : locations.getValue().getFiles()) {
+ CarbonFile carbonFile = FileFactory
+ .getCarbonFile(location + CarbonCommonConstants.FILE_SEPARATOR + indexFile);
+ if (carbonFile.exists()) {
+ carbonIndexFiles.add(carbonFile);
+ }
+ }
+ }
+ }
+ for (int i = 0; i < carbonIndexFiles.size(); i++) {
+ if (carbonIndexFiles.get(i).getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
+ readMergeFile(carbonIndexFiles.get(i).getCanonicalPath());
+ } else if (carbonIndexFiles.get(i).getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
+ readIndexFile(carbonIndexFiles.get(i));
+ }
+ }
+ }
+
+ /**
* read index file and fill the blocklet information
*
* @param segmentPath
@@ -120,17 +167,22 @@ public class SegmentIndexFileStore {
* @return
* @throws IOException
*/
- public List<String> getIndexFilesFromSegment(String segmentPath) throws IOException {
+ public Map<String, String> getIndexFilesFromSegment(String segmentPath) throws IOException {
CarbonFile[] carbonIndexFiles = getCarbonIndexFiles(segmentPath);
- Set<String> indexFiles = new HashSet<>();
+ Map<String, String> indexFiles = new HashMap<>();
for (int i = 0; i < carbonIndexFiles.length; i++) {
if (carbonIndexFiles[i].getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
- indexFiles.addAll(getIndexFilesFromMergeFile(carbonIndexFiles[i].getCanonicalPath()));
+ List<String> indexFilesFromMergeFile =
+ getIndexFilesFromMergeFile(carbonIndexFiles[i].getCanonicalPath());
+ for (String file: indexFilesFromMergeFile) {
+ indexFiles.put(carbonIndexFiles[i].getParentFile().getAbsolutePath()
+ + CarbonCommonConstants.FILE_SEPARATOR + file, carbonIndexFiles[i].getName());
+ }
} else if (carbonIndexFiles[i].getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
- indexFiles.add(carbonIndexFiles[i].getName());
+ indexFiles.put(carbonIndexFiles[i].getAbsolutePath(), null);
}
}
- return new ArrayList<>(indexFiles);
+ return indexFiles;
}
/**
@@ -156,16 +208,23 @@ public class SegmentIndexFileStore {
*/
private void readMergeFile(String mergeFilePath) throws IOException {
ThriftReader thriftReader = new ThriftReader(mergeFilePath);
- thriftReader.open();
- MergedBlockIndexHeader indexHeader = readMergeBlockIndexHeader(thriftReader);
- MergedBlockIndex mergedBlockIndex = readMergeBlockIndex(thriftReader);
- List<String> file_names = indexHeader.getFile_names();
- List<ByteBuffer> fileData = mergedBlockIndex.getFileData();
- assert (file_names.size() == fileData.size());
- for (int i = 0; i < file_names.size(); i++) {
- carbonIndexMap.put(file_names.get(i), fileData.get(i).array());
+ try {
+ thriftReader.open();
+ MergedBlockIndexHeader indexHeader = readMergeBlockIndexHeader(thriftReader);
+ MergedBlockIndex mergedBlockIndex = readMergeBlockIndex(thriftReader);
+ List<String> file_names = indexHeader.getFile_names();
+ List<ByteBuffer> fileData = mergedBlockIndex.getFileData();
+ CarbonFile mergeFile = FileFactory.getCarbonFile(mergeFilePath);
+ assert (file_names.size() == fileData.size());
+ for (int i = 0; i < file_names.size(); i++) {
+ carbonIndexMap.put(file_names.get(i), fileData.get(i).array());
+ carbonIndexMapWithFullPath.put(
+ mergeFile.getParentFile().getAbsolutePath() + CarbonCommonConstants.FILE_SEPARATOR
+ + file_names.get(i), fileData.get(i).array());
+ }
+ } finally {
+ thriftReader.close();
}
- thriftReader.close();
}
/**
@@ -181,6 +240,9 @@ public class SegmentIndexFileStore {
byte[] bytes = new byte[(int) indexFile.getSize()];
dataInputStream.readFully(bytes);
carbonIndexMap.put(indexFile.getName(), bytes);
+ carbonIndexMapWithFullPath.put(
+ indexFile.getParentFile().getAbsolutePath() + CarbonCommonConstants.FILE_SEPARATOR
+ + indexFile.getName(), bytes);
dataInputStream.close();
}
@@ -253,6 +315,10 @@ public class SegmentIndexFileStore {
return carbonIndexMap;
}
+ public Map<String, byte[]> getCarbonIndexMapWithFullPath() {
+ return carbonIndexMapWithFullPath;
+ }
+
/**
* This method will read the index information from carbon index file
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
index cc98b03..be98f7d 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -102,21 +101,6 @@ public class HdfsFileLock extends AbstractCarbonLock {
status = true;
} catch (IOException e) {
status = false;
- } finally {
- CarbonFile carbonFile =
- FileFactory.getCarbonFile(location, FileFactory.getFileType(location));
- if (carbonFile.exists()) {
- if (carbonFile.delete()) {
- LOGGER.info("Deleted the lock file " + location);
- } else {
- LOGGER.error("Not able to delete the lock file " + location);
- status = false;
- }
- } else {
- LOGGER.error("Not able to delete the lock file because "
- + "it is not existed in location " + location);
- status = false;
- }
}
}
return status;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 4fee4c4..75ea074 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -26,7 +26,6 @@ import java.nio.channels.OverlappingFileLockException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -121,7 +120,6 @@ public class LocalFileLock extends AbstractCarbonLock {
LOGGER.info(e.getMessage());
return false;
}
-
}
/**
@@ -130,27 +128,18 @@ public class LocalFileLock extends AbstractCarbonLock {
* @return
*/
@Override public boolean unlock() {
- boolean status;
+ boolean status = false;
try {
if (null != fileLock) {
fileLock.release();
+ status = true;
}
- status = true;
} catch (IOException e) {
status = false;
} finally {
if (null != fileOutputStream) {
try {
fileOutputStream.close();
- // deleting the lock file after releasing the lock.
- CarbonFile lockFile = FileFactory
- .getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath));
- if (!lockFile.exists() || lockFile.delete()) {
- LOGGER.info("Successfully deleted the lock file " + lockFilePath);
- } else {
- LOGGER.error("Not able to delete the lock file " + lockFilePath);
- status = false;
- }
} catch (IOException e) {
LOGGER.error(e.getMessage());
}
@@ -158,5 +147,4 @@ public class LocalFileLock extends AbstractCarbonLock {
}
return status;
}
-
}
[04/16] carbondata git commit: [CARBONDATA-2149]Fix complex type data
displaying error when use DataFrame to write complex type data
Posted by gv...@apache.org.
[CARBONDATA-2149]Fix complex type data displaying error when use DataFrame to write complex type data
The default value of 'complex_delimiter_level_1' and 'complex_delimiter_level_2' is wrong, it must be '$' and ':', not be '$' and '\:'. Escape characters '\' need to be added only when using delimiters in ArrayParserImpl or StructParserImpl
This closes #1962
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6f9016db
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6f9016db
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6f9016db
Branch: refs/heads/branch-1.3
Commit: 6f9016db52dd3f9c31ba20e585debfc283e2594e
Parents: 4bbbd4b
Author: Zhang Zhichao <44...@qq.com>
Authored: Fri Feb 9 17:32:54 2018 +0800
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Tue Feb 27 12:50:55 2018 +0530
----------------------------------------------------------------------
.../examples/DataFrameComplexTypeExample.scala | 90 ++++++++++++++++++++
.../hadoop/api/CarbonTableOutputFormat.java | 4 +-
.../dataload/TestLoadDataWithNoMeasure.scala | 5 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 4 +-
.../carbondata/spark/util/DataLoadingUtil.scala | 10 +--
.../spark/util/GlobalDictionaryUtil.scala | 4 +-
.../spark/util/AllDictionaryTestCase.scala | 4 +-
.../util/ExternalColumnDictionaryTestCase.scala | 4 +-
8 files changed, 107 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f9016db/examples/spark2/src/main/scala/org/apache/carbondata/examples/DataFrameComplexTypeExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/DataFrameComplexTypeExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/DataFrameComplexTypeExample.scala
new file mode 100644
index 0000000..b5ff49b
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/DataFrameComplexTypeExample.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import org.apache.spark.sql.SaveMode
+
+case class StructElement(school: Array[String], age: Int)
+case class ComplexTypeData(id: Int, name: String, city: String, salary: Float, file: StructElement)
+
+// scalastyle:off println
+object DataFrameComplexTypeExample {
+
+ def main(args: Array[String]) {
+
+ val spark = ExampleUtils.createCarbonSession("DataFrameComplexTypeExample", 4)
+ val complexTableName = s"complex_type_table"
+
+ import spark.implicits._
+
+ // drop table if exists previously
+ spark.sql(s"DROP TABLE IF EXISTS ${ complexTableName }")
+ spark.sql(
+ s"""
+ | CREATE TABLE ${ complexTableName }(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | file struct<school:array<string>, age:int>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES(
+ | 'sort_columns'='name',
+ | 'dictionary_include'='city')
+ | """.stripMargin)
+
+ val sc = spark.sparkContext
+ // generate data
+ val df = sc.parallelize(Seq(
+ ComplexTypeData(1, "index_1", "city_1", 10000.0f,
+ StructElement(Array("struct_11", "struct_12"), 10)),
+ ComplexTypeData(2, "index_2", "city_2", 20000.0f,
+ StructElement(Array("struct_21", "struct_22"), 20)),
+ ComplexTypeData(3, "index_3", "city_3", 30000.0f,
+ StructElement(Array("struct_31", "struct_32"), 30))
+ )).toDF
+ df.printSchema()
+ df.write
+ .format("carbondata")
+ .option("tableName", complexTableName)
+ .mode(SaveMode.Append)
+ .save()
+
+ spark.sql(s"select count(*) from ${ complexTableName }").show(100, truncate = false)
+
+ spark.sql(s"select * from ${ complexTableName } order by id desc").show(300, truncate = false)
+
+ spark.sql(s"select * " +
+ s"from ${ complexTableName } " +
+ s"where id = 100000001 or id = 1 limit 100").show(100, truncate = false)
+
+ spark.sql(s"select * " +
+ s"from ${ complexTableName } " +
+ s"where id > 10 limit 100").show(100, truncate = false)
+
+ // show segments
+ spark.sql(s"SHOW SEGMENTS FOR TABLE ${complexTableName}").show(false)
+
+ // drop table
+ spark.sql(s"DROP TABLE IF EXISTS ${ complexTableName }")
+
+ spark.stop()
+ }
+}
+// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f9016db/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index e600f0c..47c8da9 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -299,11 +299,11 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Stri
SKIP_EMPTY_LINE,
carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
- String complexDelim = conf.get(COMPLEX_DELIMITERS, "\\$" + "," + "\\:");
+ String complexDelim = conf.get(COMPLEX_DELIMITERS, "$" + "," + ":");
String[] split = complexDelim.split(",");
model.setComplexDelimiterLevel1(split[0]);
if (split.length > 1) {
- model.setComplexDelimiterLevel1(split[1]);
+ model.setComplexDelimiterLevel2(split[1]);
}
model.setDateFormat(
conf.get(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f9016db/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
index f0b5fe6..f1d25e6 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
@@ -86,7 +86,8 @@ class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll {
)
val testData = s"$resourcesPath/datasingleComplexCol.csv"
sql("LOAD DATA LOCAL INPATH '" + testData + "' into table nomeasureTest_scd options " +
- "('DELIMITER'=',','QUOTECHAR'='\"','FILEHEADER'='cityDetail','COMPLEX_DELIMITER_LEVEL_1'=':')"
+ "('DELIMITER'=',','QUOTECHAR'='\"','FILEHEADER'='cityDetail'," +
+ "'COMPLEX_DELIMITER_LEVEL_1'=':','COMPLEX_DELIMITER_LEVEL_2'='$')"
)
}
@@ -101,7 +102,7 @@ class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll {
val testData = s"$resourcesPath/datasingleComplexCol.csv"
sql("LOAD DATA LOCAL INPATH '" + testData + "' into table nomeasureTest_scd options " +
"('DELIMITER'=',','QUOTECHAR'='\"','FILEHEADER'='cityDetail'," +
- "'COMPLEX_DELIMITER_LEVEL_1'=':')"
+ "'COMPLEX_DELIMITER_LEVEL_1'=':','COMPLEX_DELIMITER_LEVEL_2'='$')"
)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f9016db/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 262adf2..748945d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -137,7 +137,7 @@ object CarbonScalaUtil {
builder.append(getString(x, serializationNullFormat, delimiterLevel1,
delimiterLevel2, timeStampFormat, dateFormat, level + 1)).append(delimiter)
}
- builder.substring(0, builder.length - 1)
+ builder.substring(0, builder.length - delimiter.length())
case m: scala.collection.Map[Any, Any] =>
throw new Exception("Unsupported data type: Map")
case r: org.apache.spark.sql.Row =>
@@ -151,7 +151,7 @@ object CarbonScalaUtil {
builder.append(getString(r(i), serializationNullFormat, delimiterLevel1,
delimiterLevel2, timeStampFormat, dateFormat, level + 1)).append(delimiter)
}
- builder.substring(0, builder.length - 1)
+ builder.substring(0, builder.length - delimiter.length())
case other => other.toString
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f9016db/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 3696e23..a38eaba 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -122,11 +122,11 @@ object DataLoadingUtil {
optionsFinal.put(
"complex_delimiter_level_1",
- options.getOrElse("complex_delimiter_level_1", "\\$"))
+ options.getOrElse("complex_delimiter_level_1", "$"))
optionsFinal.put(
"complex_delimiter_level_2",
- options.getOrElse("complex_delimiter_level_2", "\\:"))
+ options.getOrElse("complex_delimiter_level_2", ":"))
optionsFinal.put(
"dateformat",
@@ -323,10 +323,8 @@ object DataLoadingUtil {
delimeter.equalsIgnoreCase(complex_delimeter_level2)) {
CarbonException.analysisException(s"Field Delimiter and Complex types delimiter are same")
} else {
- carbonLoadModel.setComplexDelimiterLevel1(
- CarbonUtil.delimiterConverter(complex_delimeter_level1))
- carbonLoadModel.setComplexDelimiterLevel2(
- CarbonUtil.delimiterConverter(complex_delimeter_level2))
+ carbonLoadModel.setComplexDelimiterLevel1(complex_delimeter_level1)
+ carbonLoadModel.setComplexDelimiterLevel2(complex_delimeter_level2)
}
// set local dictionary path, and dictionary file extension
carbonLoadModel.setAllDictPath(all_dictionary_path)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f9016db/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 5f44e43..9e1ece7 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -274,10 +274,10 @@ object GlobalDictionaryUtil {
Pattern.compile(if (d == null) {
""
} else {
- d
+ CarbonUtil.delimiterConverter(d)
})
}
- DataFormat(delimiters, 0, patterns)
+ DataFormat(delimiters.map(CarbonUtil.delimiterConverter(_)), 0, patterns)
} else {
null
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f9016db/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index 56c5747..e3678cd 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -54,8 +54,8 @@ class AllDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll {
carbonLoadModel.setFactFilePath(filePath)
carbonLoadModel.setCsvHeader(header)
carbonLoadModel.setCsvDelimiter(",")
- carbonLoadModel.setComplexDelimiterLevel1("\\$")
- carbonLoadModel.setComplexDelimiterLevel2("\\:")
+ carbonLoadModel.setComplexDelimiterLevel1("$")
+ carbonLoadModel.setComplexDelimiterLevel2(":")
carbonLoadModel.setAllDictPath(allDictFilePath)
carbonLoadModel.setSerializationNullFormat(
TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f9016db/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index a1b39d8..e543893 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -163,8 +163,8 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
carbonLoadModel.setFactFilePath(filePath)
carbonLoadModel.setCsvHeader(header)
carbonLoadModel.setCsvDelimiter(csvDelimiter)
- carbonLoadModel.setComplexDelimiterLevel1("\\$")
- carbonLoadModel.setComplexDelimiterLevel2("\\:")
+ carbonLoadModel.setComplexDelimiterLevel1("$")
+ carbonLoadModel.setComplexDelimiterLevel2(":")
carbonLoadModel.setColDictFilePath(extColFilePath)
carbonLoadModel.setQuoteChar("\"");
carbonLoadModel.setSerializationNullFormat(
[08/16] carbondata git commit: [CARBONDATA-2168] Support global sort
for standard hive partitioning
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
index 4806f9f..9ea58a9 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
@@ -17,6 +17,8 @@
package org.apache.carbondata.spark.rdd
+import java.util
+
import scala.collection.JavaConverters._
import org.apache.spark.{Partition, SparkContext, TaskContext}
@@ -59,10 +61,12 @@ class CarbonDropPartitionRDD(
val iter = new Iterator[String] {
val split = theSplit.asInstanceOf[CarbonDropPartition]
logInfo("Dropping partition information from : " + split.segmentPath)
-
+ partitions.toList.asJava
+ val partitionList = new util.ArrayList[util.List[String]]()
+ partitionList.add(partitions.toList.asJava)
new PartitionMapFileStore().dropPartitions(
split.segmentPath,
- partitions.toList.asJava,
+ partitionList,
uniqueId,
partialMatch)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 748945d..73be3c8 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -26,14 +26,18 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTablePartition
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.execution.command.DataTypeInfo
import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
import org.apache.carbondata.common.constants.LoggerAction
+import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
+import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory
import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, StructField => CarbonStructField}
+import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.core.util.{ByteUtil, CarbonSessionInfo}
-import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
+import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, ColumnSchema}
+import org.apache.carbondata.core.util.{CarbonSessionInfo, DataTypeUtil}
object CarbonScalaUtil {
def convertSparkToCarbonDataType(dataType: DataType): CarbonDataType = {
@@ -196,21 +200,85 @@ object CarbonScalaUtil {
/**
* Converts incoming value to String after converting data as per the data type.
* @param value Input value to convert
- * @param dataType Datatype to convert and then convert to String
- * @param timeStampFormat Timestamp format to convert in case of timestamp datatypes
- * @param dateFormat DataFormat to convert in case of DateType datatype
+ * @param column column which it value belongs to
* @return converted String
*/
- def convertToCarbonFormat(value: String,
- dataType: DataType,
- timeStampFormat: SimpleDateFormat,
- dateFormat: SimpleDateFormat): String = {
+ def convertToCarbonFormat(
+ value: String,
+ column: CarbonColumn,
+ forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary],
+ table: CarbonTable): String = {
+ if (column.hasEncoding(Encoding.DICTIONARY)) {
+ if (column.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (column.getDataType.equals(CarbonDataTypes.TIMESTAMP)) {
+ val time = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(
+ column.getDataType,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
+ ).getValueFromSurrogate(value.toInt).toString
+ return DateTimeUtils.timestampToString(time.toLong * 1000)
+ } else if (column.getDataType.equals(CarbonDataTypes.DATE)) {
+ val date = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(
+ column.getDataType,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT
+ ).getValueFromSurrogate(value.toInt).toString
+ return DateTimeUtils.dateToString(date.toInt)
+ }
+ }
+ val dictionaryPath =
+ table.getTableInfo.getFactTable.getTableProperties.get(
+ CarbonCommonConstants.DICTIONARY_PATH)
+ val dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(
+ table.getAbsoluteTableIdentifier,
+ column.getColumnIdentifier, column.getDataType,
+ dictionaryPath)
+ return forwardDictionaryCache.get(
+ dictionaryColumnUniqueIdentifier).getDictionaryValueForKey(value.toInt)
+ }
try {
- dataType match {
- case TimestampType =>
- timeStampFormat.format(DateTimeUtils.stringToTime(value))
- case DateType =>
- dateFormat.format(DateTimeUtils.stringToTime(value))
+ column.getDataType match {
+ case CarbonDataTypes.TIMESTAMP =>
+ DateTimeUtils.timestampToString(value.toLong * 1000)
+ case CarbonDataTypes.DATE =>
+ DateTimeUtils.dateToString(DateTimeUtils.millisToDays(value.toLong))
+ case _ => value
+ }
+ } catch {
+ case e: Exception =>
+ value
+ }
+ }
+
+ /**
+ * Converts incoming value to String after converting data as per the data type.
+ * @param value Input value to convert
+ * @param column column which it value belongs to
+ * @return converted String
+ */
+ def convertStaticPartitions(
+ value: String,
+ column: ColumnSchema,
+ table: CarbonTable): String = {
+ try {
+ if (column.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (column.getDataType.equals(CarbonDataTypes.TIMESTAMP)) {
+ return DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(
+ column.getDataType,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
+ ).generateDirectSurrogateKey(value).toString
+ } else if (column.getDataType.equals(CarbonDataTypes.DATE)) {
+ return DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(
+ column.getDataType,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT
+ ).generateDirectSurrogateKey(value).toString
+ }
+ }
+ column.getDataType match {
+ case CarbonDataTypes.TIMESTAMP =>
+ DataTypeUtil.getDataDataTypeForNoDictionaryColumn(value,
+ column.getDataType,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT).toString
+ case CarbonDataTypes.DATE =>
+ DateTimeUtils.stringToDate(UTF8String.fromString(value)).get.toString
case _ => value
}
} catch {
@@ -229,11 +297,11 @@ object CarbonScalaUtil {
partitionSpec: Map[String, String],
table: CarbonTable,
timeFormat: SimpleDateFormat,
- dateFormat: SimpleDateFormat,
- serializationNullFormat: String,
- badRecordAction: String,
- isEmptyBadRecord: Boolean): Map[String, String] = {
+ dateFormat: SimpleDateFormat): Map[String, String] = {
val hivedefaultpartition = "__HIVE_DEFAULT_PARTITION__"
+ val cacheProvider: CacheProvider = CacheProvider.getInstance
+ val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] =
+ cacheProvider.createCache(CacheType.FORWARD_DICTIONARY)
partitionSpec.map{ case (col, pvalue) =>
// replace special string with empty value.
val value = if (pvalue == null) {
@@ -246,17 +314,15 @@ object CarbonScalaUtil {
val carbonColumn = table.getColumnByName(table.getTableName, col.toLowerCase)
val dataType = CarbonScalaUtil.convertCarbonToSparkDataType(carbonColumn.getDataType)
try {
- if (isEmptyBadRecord && value.length == 0 &&
- badRecordAction.equalsIgnoreCase(LoggerAction.IGNORE.toString) &&
- dataType != StringType) {
- (col, hiveignorepartition)
- } else if (!isEmptyBadRecord && value.length == 0 && dataType != StringType) {
- (col, hivedefaultpartition)
- } else if (value.equals(hivedefaultpartition)) {
+ if (value.equals(hivedefaultpartition)) {
(col, value)
} else {
- val convertedString = CarbonScalaUtil.convertToString(
- value, dataType, timeFormat, dateFormat, serializationNullFormat)
+ val convertedString =
+ CarbonScalaUtil.convertToCarbonFormat(
+ value,
+ carbonColumn,
+ forwardDictionaryCache,
+ table)
if (convertedString == null) {
(col, hivedefaultpartition)
} else {
@@ -265,13 +331,7 @@ object CarbonScalaUtil {
}
} catch {
case e: Exception =>
- // If it is bad record ignore case then add with special string so that it will be
- // filtered after this.
- if (badRecordAction.equalsIgnoreCase(LoggerAction.IGNORE.toString)) {
- (col, hiveignorepartition)
- } else {
- (col, hivedefaultpartition)
- }
+ (col, value)
}
}
}
@@ -306,10 +366,7 @@ object CarbonScalaUtil {
f.spec,
table,
timeFormat,
- dateFormat,
- serializeFormat,
- badRecordAction,
- isEmptyBadRecord)
+ dateFormat)
f.copy(spec = changedSpec)
}.filterNot{ p =>
// Filter the special bad record ignore case string
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 7d49c11..9bdaddb 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -23,6 +23,7 @@ import java.util.UUID
import scala.collection.JavaConverters._
import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.conf.Configuration
@@ -32,15 +33,15 @@ import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{NoSuchTableException, UnresolvedAttribute}
import org.apache.spark.sql.catalyst.catalog.CatalogTable
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, GenericInternalRow}
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, Expression, GenericInternalRow, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, Sort}
import org.apache.spark.sql.execution.LogicalRDD
import org.apache.spark.sql.execution.SQLExecution.EXECUTION_ID_KEY
import org.apache.spark.sql.execution.command.{AtomicRunnableCommand, DataLoadTableFileMapping, UpdateTableModel}
import org.apache.spark.sql.execution.datasources.{CarbonFileFormat, CatalogFileIndex, FindDataSourceTable, HadoopFsRelation, LogicalRelation}
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.optimizer.CarbonFilters
-import org.apache.spark.sql.types.{StringType, StructField, StructType}
+import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
@@ -55,7 +56,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, DataTypeUtil}
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.events.exception.PreEventException
@@ -65,12 +66,14 @@ import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadMetadataEvent, LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
import org.apache.carbondata.processing.loading.exception.NoRetryException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
-import org.apache.carbondata.processing.util.CarbonLoaderUtil
+import org.apache.carbondata.processing.loading.sort.SortScopeOptions
+import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil, CarbonLoaderUtil}
import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.spark.load.DataLoadProcessorStepOnSpark
import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD}
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, DataLoadingUtil, GlobalDictionaryUtil}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, DataLoadingUtil, GlobalDictionaryUtil, SparkDataTypeConverterImpl}
case class CarbonLoadDataCommand(
databaseNameOp: Option[String],
@@ -505,7 +508,7 @@ case class CarbonLoadDataCommand(
carbonLoadModel: CarbonLoadModel,
hadoopConf: Configuration,
dataFrame: Option[DataFrame],
- operationContext: OperationContext) = {
+ operationContext: OperationContext): Unit = {
val table = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val identifier = TableIdentifier(table.getTableName, Some(table.getDatabaseName))
val catalogTable: CatalogTable = logicalPartitionRelation.catalogTable.get
@@ -544,17 +547,76 @@ case class CarbonLoadDataCommand(
CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
isEmptyBadRecord)
CarbonSession.threadSet("partition.operationcontext", operationContext)
+ // input data from csv files. Convert to logical plan
+ val allCols = new ArrayBuffer[String]()
+ allCols ++= table.getAllDimensions.asScala.map(_.getColName)
+ allCols ++= table.getAllMeasures.asScala.map(_.getColName)
+ var attributes =
+ StructType(allCols.map(StructField(_, StringType))).toAttributes
+
+ var partitionsLen = 0
+ val sortScope = CarbonDataProcessorUtil.getSortScope(carbonLoadModel.getSortScope)
+ def transformQuery(rdd: RDD[Row], isDataFrame: Boolean) = {
+ val updatedRdd = convertData(rdd, sparkSession, carbonLoadModel, isDataFrame)
+ val catalogAttributes = catalogTable.schema.toAttributes
+ attributes = attributes.map(a => {
+ catalogAttributes.find(_.name.equalsIgnoreCase(a.name)).get
+ })
+ attributes = attributes.map { attr =>
+ val column = table.getColumnByName(table.getTableName, attr.name)
+ if (column.hasEncoding(Encoding.DICTIONARY)) {
+ AttributeReference(
+ attr.name,
+ IntegerType,
+ attr.nullable,
+ attr.metadata)(attr.exprId, attr.qualifier, attr.isGenerated)
+ } else if (attr.dataType == TimestampType || attr.dataType == DateType) {
+ AttributeReference(
+ attr.name,
+ LongType,
+ attr.nullable,
+ attr.metadata)(attr.exprId, attr.qualifier, attr.isGenerated)
+ } else {
+ attr
+ }
+ }
+ // Only select the required columns
+ val output = if (partition.nonEmpty) {
+ val lowerCasePartition = partition.map { case (key, value) => (key.toLowerCase, value) }
+ catalogTable.schema.map { attr =>
+ attributes.find(_.name.equalsIgnoreCase(attr.name)).get
+ }.filter(attr => lowerCasePartition.getOrElse(attr.name.toLowerCase, None).isEmpty)
+ } else {
+ catalogTable.schema.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get)
+ }
+ partitionsLen = rdd.partitions.length
+ val child = Project(output, LogicalRDD(attributes, updatedRdd)(sparkSession))
+ if (sortScope == SortScopeOptions.SortScope.GLOBAL_SORT) {
+ val sortColumns = table.getSortColumns(table.getTableName)
+ Sort(output.filter(f => sortColumns.contains(f.name)).map(SortOrder(_, Ascending)),
+ true,
+ child)
+ } else {
+ child
+ }
+ }
+
try {
val query: LogicalPlan = if (dataFrame.isDefined) {
val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
- val attributes =
+ val dfAttributes =
StructType(dataFrame.get.schema.fields.map(_.copy(dataType = StringType))).toAttributes
- val len = attributes.length
+ val partitionValues = if (partition.nonEmpty) {
+ partition.values.filter(_.nonEmpty).map(_.get).toArray
+ } else {
+ Array[String]()
+ }
+ val len = dfAttributes.length
val rdd = dataFrame.get.rdd.map { f =>
val data = new Array[Any](len)
var i = 0
- while (i < len) {
+ while (i < f.length) {
data(i) =
UTF8String.fromString(
CarbonScalaUtil.getString(f.get(i),
@@ -565,20 +627,32 @@ case class CarbonLoadDataCommand(
dateFormat))
i = i + 1
}
- InternalRow.fromSeq(data)
+ if (partitionValues.length > 0) {
+ var j = 0
+ while (i < len) {
+ data(i) = UTF8String.fromString(partitionValues(j))
+ j = j + 1
+ i = i + 1
+ }
+ }
+ Row.fromSeq(data)
}
- if (updateModel.isDefined) {
+ val transRdd = if (updateModel.isDefined) {
// Get the updated query plan in case of update scenario
- getLogicalQueryForUpdate(sparkSession, catalogTable, attributes, rdd)
+ Dataset.ofRows(
+ sparkSession,
+ getLogicalQueryForUpdate(
+ sparkSession,
+ catalogTable,
+ dfAttributes,
+ rdd.map(row => InternalRow.fromSeq(row.toSeq)),
+ carbonLoadModel)).rdd
} else {
- LogicalRDD(attributes, rdd)(sparkSession)
+ rdd
}
-
+ transformQuery(transRdd, true)
} else {
- // input data from csv files. Convert to logical plan
- val attributes =
- StructType(carbonLoadModel.getCsvHeaderColumns.map(
- StructField(_, StringType))).toAttributes
+
val rowDataTypes = attributes.map { attribute =>
catalogTable.schema.find(_.name.equalsIgnoreCase(attribute.name)) match {
case Some(attr) => attr.dataType
@@ -592,41 +666,12 @@ case class CarbonLoadDataCommand(
case _ => false
}
}
- val len = rowDataTypes.length
- var rdd =
- DataLoadingUtil.csvFileScanRDD(
- sparkSession,
- model = carbonLoadModel,
- hadoopConf)
- .map { row =>
- val data = new Array[Any](len)
- var i = 0
- val input = row.asInstanceOf[GenericInternalRow].values.asInstanceOf[Array[String]]
- val inputLen = Math.min(input.length, len)
- while (i < inputLen) {
- data(i) = UTF8String.fromString(input(i))
- // If partition column then update empty value with special string otherwise spark
- // makes it as null so we cannot internally handle badrecords.
- if (partitionColumns(i)) {
- if (input(i) != null && input(i).isEmpty) {
- data(i) = UTF8String.fromString(CarbonCommonConstants.MEMBER_DEFAULT_VAL)
- }
- }
- i = i + 1
- }
- InternalRow.fromSeq(data)
-
- }
- // Only select the required columns
- val output = if (partition.nonEmpty) {
- val lowerCasePartition = partition.map{case(key, value) => (key.toLowerCase, value)}
- catalogTable.schema.map { attr =>
- attributes.find(_.name.equalsIgnoreCase(attr.name)).get
- }.filter(attr => lowerCasePartition.getOrElse(attr.name.toLowerCase, None).isEmpty)
- } else {
- catalogTable.schema.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get)
- }
- Project(output, LogicalRDD(attributes, rdd)(sparkSession))
+ val columnCount = carbonLoadModel.getCsvHeaderColumns.length
+ var rdd = DataLoadingUtil.csvFileScanRDD(
+ sparkSession,
+ model = carbonLoadModel,
+ hadoopConf).map(DataLoadProcessorStepOnSpark.toStringArrayRow(_, columnCount))
+ transformQuery(rdd.asInstanceOf[RDD[Row]], false)
}
val convertRelation = convertToLogicalRelation(
catalogTable,
@@ -635,24 +680,29 @@ case class CarbonLoadDataCommand(
carbonLoadModel,
sparkSession,
operationContext)
+ val logicalPlan = if (sortScope == SortScopeOptions.SortScope.GLOBAL_SORT) {
+ var numPartitions =
+ CarbonDataProcessorUtil.getGlobalSortPartitions(carbonLoadModel.getGlobalSortPartitions)
+ if (numPartitions <= 0) {
+ numPartitions = partitionsLen
+ }
+ if (numPartitions > 0) {
+ Dataset.ofRows(sparkSession, query).repartition(numPartitions).logicalPlan
+ } else {
+ query
+ }
+ } else {
+ query
+ }
+
val convertedPlan =
CarbonReflectionUtils.getInsertIntoCommand(
table = convertRelation,
partition = partition,
- query = query,
+ query = logicalPlan,
overwrite = false,
ifPartitionNotExists = false)
- if (isOverwriteTable && partition.nonEmpty) {
- overwritePartition(
- sparkSession,
- table,
- convertedPlan,
- serializationNullFormat,
- badRecordAction,
- isEmptyBadRecord.toBoolean)
- } else {
- Dataset.ofRows(sparkSession, convertedPlan)
- }
+ Dataset.ofRows(sparkSession, convertedPlan)
} finally {
CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT)
@@ -660,6 +710,14 @@ case class CarbonLoadDataCommand(
CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
CarbonSession.threadUnset("partition.operationcontext")
+ if (isOverwriteTable) {
+ DataMapStoreManager.getInstance().clearDataMaps(table.getAbsoluteTableIdentifier)
+ // Clean the overwriting segments if any.
+ new PartitionMapFileStore().cleanSegments(
+ table,
+ CarbonFilters.getPartitions(Seq.empty, sparkSession, identifier).asJava,
+ false)
+ }
}
try {
// Trigger auto compaction
@@ -676,6 +734,48 @@ case class CarbonLoadDataCommand(
}
}
+ private def convertData(
+ originRDD: RDD[Row],
+ sparkSession: SparkSession,
+ model: CarbonLoadModel,
+ isDataFrame: Boolean): RDD[InternalRow] = {
+ model.setPartitionId("0")
+ val sc = sparkSession.sparkContext
+ val modelBroadcast = sc.broadcast(model)
+ val partialSuccessAccum = sc.accumulator(0, "Partial Success Accumulator")
+
+ val inputStepRowCounter = sc.accumulator(0, "Input Processor Accumulator")
+ // 1. Input
+ var convertRDD =
+ if (isDataFrame) {
+ originRDD.mapPartitions{rows =>
+ DataLoadProcessorStepOnSpark.toRDDIterator(rows, modelBroadcast)
+ }
+ } else {
+ originRDD.map{row =>
+ val array = new Array[AnyRef](row.length)
+ var i = 0
+ while (i < array.length) {
+ array(i) = row.get(i).asInstanceOf[AnyRef]
+ i = i + 1
+ }
+ array
+ }
+ }
+ val finalRDD = convertRDD.mapPartitionsWithIndex { case (index, rows) =>
+ DataTypeUtil.setDataTypeConverter(new SparkDataTypeConverterImpl)
+ DataLoadProcessorStepOnSpark.inputAndconvertFunc(
+ rows,
+ index,
+ modelBroadcast,
+ partialSuccessAccum,
+ inputStepRowCounter,
+ keepActualData = true)
+ }.filter(_ != null).map(row => InternalRow.fromSeq(row.getData))
+
+ finalRDD
+ }
+
/**
* Create the logical plan for update scenario. Here we should drop the segmentid column from the
* input rdd.
@@ -684,7 +784,8 @@ case class CarbonLoadDataCommand(
sparkSession: SparkSession,
catalogTable: CatalogTable,
attributes: Seq[AttributeReference],
- rdd: RDD[InternalRow]): LogicalPlan = {
+ rdd: RDD[InternalRow],
+ carbonLoadModel: CarbonLoadModel): LogicalPlan = {
sparkSession.sparkContext.setLocalProperty(EXECUTION_ID_KEY, null)
// In case of update, we don't need the segmrntid column in case of partitioning
val dropAttributes = attributes.dropRight(1)
@@ -698,6 +799,8 @@ case class CarbonLoadDataCommand(
}
}.get
}
+ carbonLoadModel.setCsvHeader(catalogTable.schema.map(_.name.toLowerCase).mkString(","))
+ carbonLoadModel.setCsvHeaderColumns(carbonLoadModel.getCsvHeader.split(","))
Project(finalOutput, LogicalRDD(attributes, rdd)(sparkSession))
}
@@ -709,7 +812,16 @@ case class CarbonLoadDataCommand(
sparkSession: SparkSession,
operationContext: OperationContext): LogicalRelation = {
val table = loadModel.getCarbonDataLoadSchema.getCarbonTable
- val metastoreSchema = StructType(catalogTable.schema.fields.map(_.copy(dataType = StringType)))
+ val metastoreSchema = StructType(catalogTable.schema.fields.map{f =>
+ val column = table.getColumnByName(table.getTableName, f.name)
+ if (column.hasEncoding(Encoding.DICTIONARY)) {
+ f.copy(dataType = IntegerType)
+ } else if (f.dataType == TimestampType || f.dataType == DateType) {
+ f.copy(dataType = LongType)
+ } else {
+ f
+ }
+ })
val lazyPruningEnabled = sparkSession.sqlContext.conf.manageFilesourcePartitions
val catalog = new CatalogFileIndex(
sparkSession, catalogTable, sizeInBytes)
@@ -718,20 +830,18 @@ case class CarbonLoadDataCommand(
} else {
catalog.filterPartitions(Nil) // materialize all the partitions in memory
}
- val partitionSchema =
+ var partitionSchema =
StructType(table.getPartitionInfo(table.getTableName).getColumnSchemaList.asScala.map(field =>
metastoreSchema.fields.find(_.name.equalsIgnoreCase(field.getColumnName))).map(_.get))
- val overWriteLocal = if (overWrite && partition.nonEmpty) {
- false
- } else {
- overWrite
- }
val dataSchema =
StructType(metastoreSchema
- .filterNot(field => partitionSchema.contains(field.name)))
+ .filterNot(field => partitionSchema.contains(field)))
+ if (partition.nonEmpty) {
+ partitionSchema = StructType(partitionSchema.fields.map(_.copy(dataType = StringType)))
+ }
val options = new mutable.HashMap[String, String]()
options ++= catalogTable.storage.properties
- options += (("overwrite", overWriteLocal.toString))
+ options += (("overwrite", overWrite.toString))
options += (("onepass", loadModel.getUseOnePass.toString))
options += (("dicthost", loadModel.getDictionaryServerHost))
options += (("dictport", loadModel.getDictionaryServerPort.toString))
@@ -761,108 +871,6 @@ case class CarbonLoadDataCommand(
Some(catalogTable))
}
- /**
- * Overwrite the partition data if static partitions are specified.
- * @param sparkSession
- * @param table
- * @param logicalPlan
- */
- private def overwritePartition(
- sparkSession: SparkSession,
- table: CarbonTable,
- logicalPlan: LogicalPlan,
- serializationNullFormat: String,
- badRecordAction: String,
- isEmptyBadRecord: Boolean): Unit = {
- val identifier = TableIdentifier(table.getTableName, Some(table.getDatabaseName))
-
- // Update the partitions as per the datatype expect for time and datetype as we
- // expect user provides the format in standard spark/hive formats.
- val updatedPartitions = CarbonScalaUtil.updatePartitions(
- partition.filter(_._2.isDefined).map(f => (f._1, f._2.get)),
- table,
- timeFormat = null,
- dateFormat = null,
- serializationNullFormat,
- badRecordAction,
- isEmptyBadRecord)
- val existingPartitions = sparkSession.sessionState.catalog.listPartitions(
- identifier,
- Some(updatedPartitions))
- val partitionNames = existingPartitions.toList.flatMap { partition =>
- partition.spec.seq.map{case (column, value) => column + "=" + value}
- }.toSet
- val uniqueId = System.currentTimeMillis().toString
- val segments = new SegmentStatusManager(
- table.getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments
- // If any existing partitions need to be overwritten then drop from partitionmap
- if (partitionNames.nonEmpty) {
- try {
- // First drop the partitions from partition mapper files of each segment
- new CarbonDropPartitionRDD(
- sparkSession.sparkContext,
- table.getTablePath,
- segments.asScala,
- partitionNames.toSeq,
- uniqueId,
- partialMatch = false).collect()
- } catch {
- case e: Exception =>
- // roll back the drop partitions from carbon store
- new CarbonDropPartitionCommitRDD(
- sparkSession.sparkContext,
- table.getTablePath,
- segments.asScala,
- success = false,
- uniqueId,
- partitionNames.toSeq).collect()
- throw e
- }
-
- try {
- Dataset.ofRows(sparkSession, logicalPlan)
- } catch {
- case e: Exception =>
- // roll back the drop partitions from carbon store
- new CarbonDropPartitionCommitRDD(
- sparkSession.sparkContext,
- table.getTablePath,
- segments.asScala,
- success = false,
- uniqueId,
- partitionNames.toSeq).collect()
- throw e
- }
- // Commit the removed partitions in carbon store.
- new CarbonDropPartitionCommitRDD(
- sparkSession.sparkContext,
- table.getTablePath,
- segments.asScala,
- success = true,
- uniqueId,
- partitionNames.toSeq).collect()
- // get valid segments
- val validsegments =
- new SegmentStatusManager(
- table.getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments
- // Update the loadstatus with update time to clear cache from driver.
- CarbonUpdateUtil.updateTableMetadataStatus(
- new util.HashSet[String](validsegments),
- table,
- uniqueId,
- true,
- new util.ArrayList[String])
- DataMapStoreManager.getInstance().clearDataMaps(table.getAbsoluteTableIdentifier)
- // Clean the overwriting segments if any.
- new PartitionMapFileStore().cleanSegments(
- table,
- CarbonFilters.getPartitions(Seq.empty, sparkSession, identifier).asJava,
- false)
- } else {
- // Otherwise its a normal load
- Dataset.ofRows(sparkSession, logicalPlan)
- }
- }
def getDataFrameWithTupleID(): DataFrame = {
val fields = dataFrame.get.schema.fields
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index 17749c8..d2c691b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -36,14 +36,17 @@ import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.DataSourceRegister
-import org.apache.spark.sql.types.{DataType, StructType}
+import org.apache.spark.sql.types._
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
import org.apache.carbondata.core.metadata.PartitionMapFileStore
-import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.metadata.encoder.Encoding
+import org.apache.carbondata.core.util.{CarbonProperties, DataTypeUtil}
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.hadoop.api.{CarbonOutputCommitter, CarbonTableOutputFormat}
import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat.CarbonRecordWriter
+import org.apache.carbondata.hadoop.internal.ObjectArrayWritable
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil
import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
@@ -110,6 +113,7 @@ with Serializable {
model.setDictionaryServerHost(options.getOrElse("dicthost", null))
model.setDictionaryServerPort(options.getOrElse("dictport", "-1").toInt)
CarbonTableOutputFormat.setOverwrite(conf, options("overwrite").toBoolean)
+ model.setPartitionLoad(true)
// Set the update timestamp if user sets in case of update query. It needs to be updated
// in load status update time
val updateTimeStamp = options.get("updatetimestamp")
@@ -231,7 +235,9 @@ private class CarbonOutputWriter(path: String,
fieldTypes: Seq[DataType],
taskNo : String)
extends OutputWriter with AbstractCarbonOutputWriter {
- val partitions = getPartitionsFromPath(path, context).map(ExternalCatalogUtils.unescapePathName)
+ val model = CarbonTableOutputFormat.getLoadModel(context.getConfiguration)
+ val partitions =
+ getPartitionsFromPath(path, context, model).map(ExternalCatalogUtils.unescapePathName)
val staticPartition: util.HashMap[String, Boolean] = {
val staticPart = context.getConfiguration.get("carbon.staticpartition")
if (staticPart != null) {
@@ -272,24 +278,42 @@ private class CarbonOutputWriter(path: String,
val formattedPartitions = updatedPartitions.map {case (col, value) =>
// Only convert the static partitions to the carbon format and use it while loading data
// to carbon.
- if (staticPartition.asScala.getOrElse(col, false)) {
- (col, CarbonScalaUtil.convertToCarbonFormat(value,
- CarbonScalaUtil.convertCarbonToSparkDataType(
- table.getColumnByName(table.getTableName, col).getDataType),
- timeFormat,
- dateFormat))
- } else {
- (col, value)
- }
+ (col, value)
}
- (formattedPartitions, formattedPartitions.map(_._2))
+ (formattedPartitions, updatePartitions(formattedPartitions.map(_._2)))
} else {
- (updatedPartitions, updatedPartitions.map(_._2))
+ (updatedPartitions, updatePartitions(updatedPartitions.map(_._2)))
}
} else {
(Map.empty[String, String].toArray, Array.empty)
}
- val writable = new StringArrayWritable()
+
+ val writable = new ObjectArrayWritable
+
+ private def updatePartitions(partitionData: Seq[String]): Array[AnyRef] = {
+ model.getCarbonDataLoadSchema.getCarbonTable.getTableInfo.getFactTable.getPartitionInfo
+ .getColumnSchemaList.asScala.zipWithIndex.map { case (col, index) =>
+
+ val dataType = if (col.hasEncoding(Encoding.DICTIONARY)) {
+ DataTypes.INT
+ } else if (col.getDataType.equals(DataTypes.TIMESTAMP) ||
+ col.getDataType.equals(DataTypes.DATE)) {
+ DataTypes.LONG
+ } else {
+ col.getDataType
+ }
+ if (staticPartition != null) {
+ DataTypeUtil.getDataBasedOnDataType(
+ CarbonScalaUtil.convertStaticPartitions(
+ partitionData(index),
+ col,
+ model.getCarbonDataLoadSchema.getCarbonTable),
+ dataType)
+ } else {
+ DataTypeUtil.getDataBasedOnDataType(partitionData(index), dataType)
+ }
+ }.toArray
+ }
private val recordWriter: CarbonRecordWriter = {
context.getConfiguration.set("carbon.outputformat.taskno", taskNo)
@@ -302,11 +326,18 @@ private class CarbonOutputWriter(path: String,
// TODO Implement writesupport interface to support writing Row directly to recordwriter
def writeCarbon(row: InternalRow): Unit = {
- val data = new Array[String](fieldTypes.length + partitionData.length)
+ val data = new Array[AnyRef](fieldTypes.length + partitionData.length)
var i = 0
while (i < fieldTypes.length) {
if (!row.isNullAt(i)) {
- data(i) = row.getString(i)
+ fieldTypes(i) match {
+ case StringType =>
+ data(i) = row.getString(i)
+ case d: DecimalType =>
+ data(i) = row.getDecimal(i, d.precision, d.scale).toJavaBigDecimal
+ case other =>
+ data(i) = row.get(i, other)
+ }
}
i += 1
}
@@ -349,10 +380,7 @@ private class CarbonOutputWriter(path: String,
updatedPartitions.toMap,
table,
timeFormat,
- dateFormat,
- serializeFormat,
- badRecordAction,
- isEmptyBadRecord)
+ dateFormat)
formattedPartitions.foreach(p => partitonList.add(p._1 + "=" + p._2))
new PartitionMapFileStore().writePartitionMapFile(
segmentPath,
@@ -360,10 +388,13 @@ private class CarbonOutputWriter(path: String,
partitonList)
}
- def getPartitionsFromPath(path: String, attemptContext: TaskAttemptContext): Array[String] = {
+ def getPartitionsFromPath(
+ path: String,
+ attemptContext: TaskAttemptContext,
+ model: CarbonLoadModel): Array[String] = {
var attemptId = attemptContext.getTaskAttemptID.toString + "/"
if (path.indexOf(attemptId) <= 0) {
- val model = CarbonTableOutputFormat.getLoadModel(attemptContext.getConfiguration)
+
attemptId = model.getTableName + "/"
}
val str = path.substring(path.indexOf(attemptId) + attemptId.length, path.lastIndexOf("/"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
index fb78deb..dc2fbbb 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
@@ -37,6 +37,8 @@ public class DataField implements Serializable {
private String timestampFormat;
+ private boolean useActualData;
+
public boolean hasDictionaryEncoding() {
return column.hasEncoding(Encoding.DICTIONARY);
}
@@ -60,4 +62,12 @@ public class DataField implements Serializable {
public void setTimestampFormat(String timestampFormat) {
this.timestampFormat = timestampFormat;
}
+
+ public boolean isUseActualData() {
+ return useActualData;
+ }
+
+ public void setUseActualData(boolean useActualData) {
+ this.useActualData = useActualData;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index f7eff81..ba24d41 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -41,6 +41,7 @@ import org.apache.carbondata.processing.loading.steps.DataConverterProcessorStep
import org.apache.carbondata.processing.loading.steps.DataConverterProcessorWithBucketingStepImpl;
import org.apache.carbondata.processing.loading.steps.DataWriterBatchProcessorStepImpl;
import org.apache.carbondata.processing.loading.steps.DataWriterProcessorStepImpl;
+import org.apache.carbondata.processing.loading.steps.InputProcessorStepForPartitionImpl;
import org.apache.carbondata.processing.loading.steps.InputProcessorStepImpl;
import org.apache.carbondata.processing.loading.steps.SortProcessorStepImpl;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -62,6 +63,8 @@ public final class DataLoadProcessBuilder {
return buildInternalForBucketing(inputIterators, configuration);
} else if (sortScope.equals(SortScopeOptions.SortScope.BATCH_SORT)) {
return buildInternalForBatchSort(inputIterators, configuration);
+ } else if (loadModel.isPartitionLoad()) {
+ return buildInternalForPartitionLoad(inputIterators, configuration, sortScope);
} else {
return buildInternal(inputIterators, configuration);
}
@@ -96,6 +99,32 @@ public final class DataLoadProcessBuilder {
return new CarbonRowDataWriterProcessorStepImpl(configuration, converterProcessorStep);
}
+ /**
+ * Build pipe line for partition load
+ */
+ private AbstractDataLoadProcessorStep buildInternalForPartitionLoad(
+ CarbonIterator[] inputIterators, CarbonDataLoadConfiguration configuration,
+ SortScopeOptions.SortScope sortScope) {
+ // Wraps with dummy processor.
+ AbstractDataLoadProcessorStep inputProcessorStep =
+ new InputProcessorStepForPartitionImpl(configuration, inputIterators);
+ if (sortScope.equals(SortScopeOptions.SortScope.LOCAL_SORT)) {
+ AbstractDataLoadProcessorStep sortProcessorStep =
+ new SortProcessorStepImpl(configuration, inputProcessorStep);
+ // Writes the sorted data in carbondata format.
+ return new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
+ } else if (sortScope.equals(SortScopeOptions.SortScope.BATCH_SORT)) {
+ // Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep =
+ new SortProcessorStepImpl(configuration, inputProcessorStep);
+ // Writes the sorted data in carbondata format.
+ return new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
+ } else {
+ // In all other cases like global sort and no sort uses this step
+ return new CarbonRowDataWriterProcessorStepImpl(configuration, inputProcessorStep);
+ }
+ }
+
private AbstractDataLoadProcessorStep buildInternalForBatchSort(CarbonIterator[] inputIterators,
CarbonDataLoadConfiguration configuration) {
// 1. Reads the data input iterators and parses the data.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
index 2d70f03..85eb19b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
@@ -47,6 +47,8 @@ public class MeasureFieldConverterImpl implements FieldConverter {
private boolean isEmptyBadRecord;
+ private DataField dataField;
+
public MeasureFieldConverterImpl(DataField dataField, String nullformat, int index,
boolean isEmptyBadRecord) {
this.dataType = dataField.getColumn().getDataType();
@@ -54,6 +56,7 @@ public class MeasureFieldConverterImpl implements FieldConverter {
this.nullformat = nullformat;
this.index = index;
this.isEmptyBadRecord = isEmptyBadRecord;
+ this.dataField = dataField;
}
@Override
@@ -85,7 +88,11 @@ public class MeasureFieldConverterImpl implements FieldConverter {
row.update(null, index);
} else {
try {
- output = DataTypeUtil.getMeasureValueBasedOnDataType(value, dataType, measure);
+ if (dataField.isUseActualData()) {
+ output = DataTypeUtil.getConvertedMeasureValueBasedOnDataType(value, dataType, measure);
+ } else {
+ output = DataTypeUtil.getMeasureValueBasedOnDataType(value, dataType, measure);
+ }
row.update(output, index);
} catch (NumberFormatException e) {
LOGGER.warn(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
index ced37dd..9cf7fe4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -68,14 +68,25 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
dateFormat = dataField.getTimestampFormat();
}
try {
- byte[] value = DataTypeUtil
- .getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType, dateFormat);
- if (dataType == DataTypes.STRING
- && value.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
- throw new CarbonDataLoadingException("Dataload failed, String size cannot exceed "
- + CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT + " bytes");
+ if (!dataField.isUseActualData()) {
+ byte[] value = DataTypeUtil
+ .getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType, dateFormat);
+ if (dataType == DataTypes.STRING
+ && value.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
+ throw new CarbonDataLoadingException("Dataload failed, String size cannot exceed "
+ + CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT + " bytes");
+ }
+ row.update(value, index);
+ } else {
+ Object value = DataTypeUtil
+ .getDataDataTypeForNoDictionaryColumn(dimensionValue, dataType, dateFormat);
+ if (dataType == DataTypes.STRING
+ && value.toString().length() > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
+ throw new CarbonDataLoadingException("Dataload failed, String size cannot exceed "
+ + CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT + " bytes");
+ }
+ row.update(value, index);
}
- row.update(value, index);
} catch (CarbonDataLoadingException e) {
throw e;
} catch (Throwable ex) {
@@ -99,7 +110,9 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
}
private void updateWithNullValue(CarbonRow row) {
- if (dataType == DataTypes.STRING) {
+ if (dataField.isUseActualData()) {
+ row.update(null, index);
+ } else if (dataType == DataTypes.STRING) {
row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, index);
} else {
row.update(CarbonCommonConstants.EMPTY_BYTE_ARRAY, index);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
index 66943c8..9229598 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/iterator/CarbonOutputIteratorWrapper.java
@@ -29,7 +29,7 @@ import org.apache.commons.logging.LogFactory;
* It is wrapper class to hold the rows in batches when record writer writes the data and allows
* to iterate on it during data load. It uses blocking queue to coordinate between read and write.
*/
-public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
+public class CarbonOutputIteratorWrapper extends CarbonIterator<Object[]> {
private static final Log LOG = LogFactory.getLog(CarbonOutputIteratorWrapper.class);
@@ -46,7 +46,7 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
private ArrayBlockingQueue<RowBatch> queue = new ArrayBlockingQueue<>(10);
- public void write(String[] row) throws InterruptedException {
+ public void write(Object[] row) throws InterruptedException {
if (!loadBatch.addRow(row)) {
loadBatch.readyRead();
queue.put(loadBatch);
@@ -78,7 +78,7 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
}
@Override
- public String[] next() {
+ public Object[] next() {
return readBatch.next();
}
@@ -100,16 +100,16 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
}
}
- private static class RowBatch extends CarbonIterator<String[]> {
+ private static class RowBatch extends CarbonIterator<Object[]> {
private int counter;
- private String[][] batch;
+ private Object[][] batch;
private int size;
private RowBatch(int size) {
- batch = new String[size][];
+ batch = new Object[size][];
this.size = size;
}
@@ -118,7 +118,7 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
* @param row
* @return false if the row cannot be added as batch is full.
*/
- public boolean addRow(String[] row) {
+ public boolean addRow(Object[] row) {
batch[counter++] = row;
return counter < size;
}
@@ -134,7 +134,7 @@ public class CarbonOutputIteratorWrapper extends CarbonIterator<String[]> {
}
@Override
- public String[] next() {
+ public Object[] next() {
assert (counter < size);
return batch[counter++];
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index d41455f..4c536ea 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -188,6 +188,11 @@ public class CarbonLoadModel implements Serializable {
private boolean isAggLoadRequest;
+ /**
+ * It directly writes data directly to nosort processor bypassing all other processors.
+ */
+ private boolean isPartitionLoad;
+
public boolean isAggLoadRequest() {
return isAggLoadRequest;
}
@@ -401,6 +406,7 @@ public class CarbonLoadModel implements Serializable {
copy.batchSortSizeInMb = batchSortSizeInMb;
copy.badRecordsLocation = badRecordsLocation;
copy.isAggLoadRequest = isAggLoadRequest;
+ copy.isPartitionLoad = isPartitionLoad;
return copy;
}
@@ -454,6 +460,7 @@ public class CarbonLoadModel implements Serializable {
copy.batchSortSizeInMb = batchSortSizeInMb;
copy.isAggLoadRequest = isAggLoadRequest;
copy.badRecordsLocation = badRecordsLocation;
+ copy.isPartitionLoad = isPartitionLoad;
return copy;
}
@@ -855,4 +862,12 @@ public class CarbonLoadModel implements Serializable {
public void setSkipEmptyLine(String skipEmptyLine) {
this.skipEmptyLine = skipEmptyLine;
}
+
+ public boolean isPartitionLoad() {
+ return isPartitionLoad;
+ }
+
+ public void setPartitionLoad(boolean partitionLoad) {
+ isPartitionLoad = partitionLoad;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepForPartitionImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepForPartitionImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepForPartitionImpl.java
new file mode 100644
index 0000000..1dc9b27
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/InputProcessorStepForPartitionImpl.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.loading.steps;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
+import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
+import org.apache.carbondata.processing.loading.DataField;
+import org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl;
+import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+
+/**
+ * It reads data from record reader and sends data to next step.
+ */
+public class InputProcessorStepForPartitionImpl extends AbstractDataLoadProcessorStep {
+
+ private CarbonIterator<Object[]>[] inputIterators;
+
+ private boolean[] noDictionaryMapping;
+
+ private DataType[] dataTypes;
+
+ private int[] orderOfData;
+
+ public InputProcessorStepForPartitionImpl(CarbonDataLoadConfiguration configuration,
+ CarbonIterator<Object[]>[] inputIterators) {
+ super(configuration, null);
+ this.inputIterators = inputIterators;
+ }
+
+ @Override public DataField[] getOutput() {
+ return configuration.getDataFields();
+ }
+
+ @Override public void initialize() throws IOException {
+ super.initialize();
+ // if logger is enabled then raw data will be required.
+ RowConverterImpl rowConverter =
+ new RowConverterImpl(configuration.getDataFields(), configuration, null);
+ rowConverter.initialize();
+ configuration.setCardinalityFinder(rowConverter);
+ noDictionaryMapping =
+ CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
+ dataTypes = new DataType[configuration.getDataFields().length];
+ for (int i = 0; i < dataTypes.length; i++) {
+ if (configuration.getDataFields()[i].getColumn().hasEncoding(Encoding.DICTIONARY)) {
+ dataTypes[i] = DataTypes.INT;
+ } else {
+ dataTypes[i] = configuration.getDataFields()[i].getColumn().getDataType();
+ }
+ }
+ orderOfData = arrangeData(configuration.getDataFields(), configuration.getHeader());
+ }
+
+ private int[] arrangeData(DataField[] dataFields, String[] header) {
+ int[] data = new int[dataFields.length];
+ for (int i = 0; i < dataFields.length; i++) {
+ for (int j = 0; j < header.length; j++) {
+ if (dataFields[i].getColumn().getColName().equalsIgnoreCase(header[j])) {
+ data[i] = j;
+ break;
+ }
+ }
+ }
+ return data;
+ }
+
+ @Override public Iterator<CarbonRowBatch>[] execute() {
+ int batchSize = CarbonProperties.getInstance().getBatchSize();
+ List<CarbonIterator<Object[]>>[] readerIterators = partitionInputReaderIterators();
+ Iterator<CarbonRowBatch>[] outIterators = new Iterator[readerIterators.length];
+ for (int i = 0; i < outIterators.length; i++) {
+ outIterators[i] =
+ new InputProcessorIterator(readerIterators[i], batchSize, configuration.isPreFetch(),
+ rowCounter, orderOfData, noDictionaryMapping, dataTypes);
+ }
+ return outIterators;
+ }
+
+ /**
+ * Partition input iterators equally as per the number of threads.
+ *
+ * @return
+ */
+ private List<CarbonIterator<Object[]>>[] partitionInputReaderIterators() {
+ // Get the number of cores configured in property.
+ int numberOfCores = CarbonProperties.getInstance().getNumberOfCores();
+ // Get the minimum of number of cores and iterators size to get the number of parallel threads
+ // to be launched.
+ int parallelThreadNumber = Math.min(inputIterators.length, numberOfCores);
+
+ List<CarbonIterator<Object[]>>[] iterators = new List[parallelThreadNumber];
+ for (int i = 0; i < parallelThreadNumber; i++) {
+ iterators[i] = new ArrayList<>();
+ }
+ // Equally partition the iterators as per number of threads
+ for (int i = 0; i < inputIterators.length; i++) {
+ iterators[i % parallelThreadNumber].add(inputIterators[i]);
+ }
+ return iterators;
+ }
+
+ @Override protected CarbonRow processRow(CarbonRow row) {
+ return null;
+ }
+
+ @Override public void close() {
+ if (!closed) {
+ super.close();
+ for (CarbonIterator inputIterator : inputIterators) {
+ inputIterator.close();
+ }
+ }
+ }
+
+ @Override protected String getStepName() {
+ return "Input Processor";
+ }
+
+ /**
+ * This iterator wraps the list of iterators and it starts iterating the each
+ * iterator of the list one by one. It also parse the data while iterating it.
+ */
+ private static class InputProcessorIterator extends CarbonIterator<CarbonRowBatch> {
+
+ private List<CarbonIterator<Object[]>> inputIterators;
+
+ private CarbonIterator<Object[]> currentIterator;
+
+ private int counter;
+
+ private int batchSize;
+
+ private boolean nextBatch;
+
+ private boolean firstTime;
+
+ private AtomicLong rowCounter;
+
+ private boolean[] noDictionaryMapping;
+
+ private DataType[] dataTypes;
+
+ private int[] orderOfData;
+
+ public InputProcessorIterator(List<CarbonIterator<Object[]>> inputIterators, int batchSize,
+ boolean preFetch, AtomicLong rowCounter, int[] orderOfData, boolean[] noDictionaryMapping,
+ DataType[] dataTypes) {
+ this.inputIterators = inputIterators;
+ this.batchSize = batchSize;
+ this.counter = 0;
+ // Get the first iterator from the list.
+ currentIterator = inputIterators.get(counter++);
+ this.rowCounter = rowCounter;
+ this.nextBatch = false;
+ this.firstTime = true;
+ this.noDictionaryMapping = noDictionaryMapping;
+ this.dataTypes = dataTypes;
+ this.orderOfData = orderOfData;
+ }
+
+ @Override public boolean hasNext() {
+ return nextBatch || internalHasNext();
+ }
+
+ private boolean internalHasNext() {
+ if (firstTime) {
+ firstTime = false;
+ currentIterator.initialize();
+ }
+ boolean hasNext = currentIterator.hasNext();
+ // If iterator is finished then check for next iterator.
+ if (!hasNext) {
+ currentIterator.close();
+ // Check next iterator is available in the list.
+ if (counter < inputIterators.size()) {
+ // Get the next iterator from the list.
+ currentIterator = inputIterators.get(counter++);
+ // Initialize the new iterator
+ currentIterator.initialize();
+ hasNext = internalHasNext();
+ }
+ }
+ return hasNext;
+ }
+
+ @Override public CarbonRowBatch next() {
+ return getBatch();
+ }
+
+ private CarbonRowBatch getBatch() {
+ // Create batch and fill it.
+ CarbonRowBatch carbonRowBatch = new CarbonRowBatch(batchSize);
+ int count = 0;
+ while (internalHasNext() && count < batchSize) {
+ carbonRowBatch.addRow(new CarbonRow(convertToNoDictionaryToBytes(currentIterator.next())));
+ count++;
+ }
+ rowCounter.getAndAdd(carbonRowBatch.getSize());
+ return carbonRowBatch;
+ }
+
+ private Object[] convertToNoDictionaryToBytes(Object[] data) {
+ Object[] newData = new Object[data.length];
+ for (int i = 0; i < noDictionaryMapping.length; i++) {
+ if (noDictionaryMapping[i]) {
+ newData[i] = DataTypeUtil
+ .getBytesDataDataTypeForNoDictionaryColumn(data[orderOfData[i]], dataTypes[i]);
+ } else {
+ newData[i] = data[orderOfData[i]];
+ }
+ }
+ if (newData.length > noDictionaryMapping.length) {
+ for (int i = noDictionaryMapping.length; i < newData.length; i++) {
+ newData[i] = data[orderOfData[i]];
+ }
+ }
+ // System.out.println(Arrays.toString(data));
+ return newData;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 06522a4..b795696 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -558,6 +558,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
@Override public Void call() throws Exception {
try {
TablePage tablePage = processDataRows(dataRows);
+ dataRows = null;
tablePage.setIsLastPage(isLastPage);
// insert the object in array according to sequence number
int indexInNodeHolderArray = (pageId - 1) % numberOfCores;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 2a4cc00..376a546 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -482,22 +482,19 @@ public final class CarbonDataProcessorUtil {
/**
* Get the number of partitions in global sort
- * @param configuration
+ * @param globalSortPartitions
* @return the number of partitions
*/
- public static int getGlobalSortPartitions(CarbonDataLoadConfiguration configuration) {
+ public static int getGlobalSortPartitions(Object globalSortPartitions) {
int numPartitions;
try {
// First try to get the number from ddl, otherwise get it from carbon properties.
- if (configuration.getDataLoadProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS)
- == null) {
+ if (globalSortPartitions == null) {
numPartitions = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT));
} else {
- numPartitions = Integer.parseInt(
- configuration.getDataLoadProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS)
- .toString());
+ numPartitions = Integer.parseInt(globalSortPartitions.toString());
}
} catch (Exception e) {
numPartitions = 0;
[09/16] carbondata git commit: [CARBONDATA-2168] Support global sort
for standard hive partitioning
Posted by gv...@apache.org.
[CARBONDATA-2168] Support global sort for standard hive partitioning
This closes #1972
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/758d03e7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/758d03e7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/758d03e7
Branch: refs/heads/branch-1.3
Commit: 758d03e783e324f70b6599be7feb1951b1034f51
Parents: 0f210c8
Author: ravipesala <ra...@gmail.com>
Authored: Fri Feb 9 09:37:02 2018 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Tue Feb 27 13:17:58 2018 +0530
----------------------------------------------------------------------
.../datastore/page/SafeDecimalColumnPage.java | 2 +-
.../datastore/page/SafeFixLengthColumnPage.java | 7 +
.../datastore/page/SafeVarLengthColumnPage.java | 1 +
.../core/metadata/PartitionMapFileStore.java | 20 +-
.../carbondata/core/util/DataTypeUtil.java | 89 +++
.../hadoop/api/CarbonOutputCommitter.java | 73 +-
.../hadoop/api/CarbonTableOutputFormat.java | 12 +-
.../hadoop/internal/ObjectArrayWritable.java | 66 ++
.../hadoop/ft/CarbonTableOutputFormatTest.java | 9 +-
.../testsuite/sortcolumns/TestSortColumns.scala | 21 +-
.../StandardPartitionBadRecordLoggerTest.scala | 3 +-
.../StandardPartitionGlobalSortTestCase.scala | 684 +++++++++++++++++++
.../StandardPartitionTableLoadingTestCase.scala | 3 +
...tandardPartitionTableOverwriteTestCase.scala | 26 +-
.../load/DataLoadProcessBuilderOnSpark.scala | 3 +-
.../load/DataLoadProcessorStepOnSpark.scala | 56 +-
.../carbondata/spark/load/ValidateUtil.scala | 1 +
.../spark/rdd/CarbonDropPartitionRDD.scala | 8 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 135 ++--
.../management/CarbonLoadDataCommand.scala | 364 +++++-----
.../datasources/CarbonFileFormat.scala | 77 ++-
.../processing/loading/DataField.java | 10 +
.../loading/DataLoadProcessBuilder.java | 29 +
.../impl/MeasureFieldConverterImpl.java | 9 +-
.../impl/NonDictionaryFieldConverterImpl.java | 29 +-
.../iterator/CarbonOutputIteratorWrapper.java | 16 +-
.../loading/model/CarbonLoadModel.java | 15 +
.../InputProcessorStepForPartitionImpl.java | 251 +++++++
.../store/CarbonFactDataHandlerColumnar.java | 1 +
.../util/CarbonDataProcessorUtil.java | 11 +-
30 files changed, 1730 insertions(+), 301 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
index cb2d3bd..0e0ba85 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
@@ -217,6 +217,6 @@ public class SafeDecimalColumnPage extends DecimalColumnPage {
@Override
public void freeMemory() {
-
+ byteArrayData = null;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
index 5f848c0..1e4445a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
@@ -311,6 +311,13 @@ public class SafeFixLengthColumnPage extends ColumnPage {
@Override
public void freeMemory() {
+ byteData = null;
+ shortData = null;
+ intData = null;
+ longData = null;
+ floatData = null;
+ doubleData = null;
+ shortIntData = null;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
index b5daddb..782b9dc 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
@@ -37,6 +37,7 @@ public class SafeVarLengthColumnPage extends VarLengthColumnPageBase {
@Override
public void freeMemory() {
+ byteArrayData = null;
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index 1e9cbc4..a0ce24a 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -279,18 +279,20 @@ public class PartitionMapFileStore {
* dropped
* @throws IOException
*/
- public void dropPartitions(String segmentPath, List<String> partitionsToDrop, String uniqueId,
- boolean partialMatch) throws IOException {
+ public void dropPartitions(String segmentPath, List<List<String>> partitionsToDrop,
+ String uniqueId, boolean partialMatch) throws IOException {
readAllPartitionsOfSegment(segmentPath);
List<String> indexesToDrop = new ArrayList<>();
for (Map.Entry<String, List<String>> entry: partitionMap.entrySet()) {
- if (partialMatch) {
- if (entry.getValue().containsAll(partitionsToDrop)) {
- indexesToDrop.add(entry.getKey());
- }
- } else {
- if (partitionsToDrop.containsAll(entry.getValue())) {
- indexesToDrop.add(entry.getKey());
+ for (List<String> partitions: partitionsToDrop) {
+ if (partialMatch) {
+ if (entry.getValue().containsAll(partitions)) {
+ indexesToDrop.add(entry.getKey());
+ }
+ } else {
+ if (partitions.containsAll(entry.getValue())) {
+ indexesToDrop.add(entry.getKey());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 6d224cf..c370b14 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -104,6 +104,38 @@ public final class DataTypeUtil {
}
}
+ /**
+ * This method will convert a given value to its specific type
+ *
+ * @param msrValue
+ * @param dataType
+ * @param carbonMeasure
+ * @return
+ */
+ public static Object getConvertedMeasureValueBasedOnDataType(String msrValue, DataType dataType,
+ CarbonMeasure carbonMeasure) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return BooleanConvert.parseBoolean(msrValue);
+ } else if (DataTypes.isDecimal(dataType)) {
+ BigDecimal bigDecimal =
+ new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
+ return converter
+ .convertToDecimal(normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision()));
+ } else if (dataType == DataTypes.SHORT) {
+ return Short.parseShort(msrValue);
+ } else if (dataType == DataTypes.INT) {
+ return Integer.parseInt(msrValue);
+ } else if (dataType == DataTypes.LONG) {
+ return Long.valueOf(msrValue);
+ } else {
+ Double parsedValue = Double.valueOf(msrValue);
+ if (Double.isInfinite(parsedValue) || Double.isNaN(parsedValue)) {
+ return null;
+ }
+ return parsedValue;
+ }
+ }
+
public static Object getMeasureObjectFromDataType(byte[] data, DataType dataType) {
if (data == null || data.length == 0) {
return null;
@@ -332,6 +364,63 @@ public final class DataTypeUtil {
}
}
+ public static Object getDataDataTypeForNoDictionaryColumn(String dimensionValue,
+ DataType actualDataType, String dateFormat) {
+ if (actualDataType == DataTypes.BOOLEAN) {
+ return BooleanConvert.parseBoolean(dimensionValue);
+ } else if (actualDataType == DataTypes.STRING) {
+ return converter.convertFromStringToUTF8String(dimensionValue);
+ } else if (actualDataType == DataTypes.SHORT) {
+ return Short.parseShort(dimensionValue);
+ } else if (actualDataType == DataTypes.INT) {
+ return Integer.parseInt(dimensionValue);
+ } else if (actualDataType == DataTypes.LONG) {
+ return Long.parseLong(dimensionValue);
+ } else if (actualDataType == DataTypes.TIMESTAMP) {
+ Date dateToStr = null;
+ DateFormat dateFormatter = null;
+ try {
+ if (null != dateFormat && !dateFormat.trim().isEmpty()) {
+ dateFormatter = new SimpleDateFormat(dateFormat);
+ } else {
+ dateFormatter = timeStampformatter.get();
+ }
+ dateToStr = dateFormatter.parse(dimensionValue);
+ return dateToStr.getTime();
+ } catch (ParseException e) {
+ throw new NumberFormatException(e.getMessage());
+ }
+ } else {
+ return converter.convertFromStringToUTF8String(dimensionValue);
+ }
+ }
+
+ public static byte[] getBytesDataDataTypeForNoDictionaryColumn(Object dimensionValue,
+ DataType actualDataType) {
+ if (dimensionValue == null) {
+ if (actualDataType == DataTypes.STRING) {
+ return CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
+ } else {
+ return new byte[0];
+ }
+ }
+ if (actualDataType == DataTypes.BOOLEAN) {
+ return ByteUtil.toBytes((Boolean) dimensionValue);
+ } else if (actualDataType == DataTypes.STRING) {
+ return ByteUtil.toBytes(dimensionValue.toString());
+ } else if (actualDataType == DataTypes.SHORT) {
+ return ByteUtil.toBytes((Short) dimensionValue);
+ } else if (actualDataType == DataTypes.INT) {
+ return ByteUtil.toBytes((Integer) dimensionValue);
+ } else if (actualDataType == DataTypes.LONG) {
+ return ByteUtil.toBytes((Long) dimensionValue);
+ } else if (actualDataType == DataTypes.TIMESTAMP) {
+ return ByteUtil.toBytes((Long)dimensionValue);
+ } else {
+ return ByteUtil.toBytes(dimensionValue.toString());
+ }
+ }
+
/**
* Below method will be used to convert the data passed to its actual data
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index 555ddd2..ce97169 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -18,10 +18,7 @@
package org.apache.carbondata.hadoop.api;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -122,7 +119,11 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
throw new IOException(e);
}
}
- CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, overwriteSet);
+ String uniqueId = null;
+ if (overwriteSet) {
+ uniqueId = overwritePartitions(loadModel);
+ }
+ CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, false);
if (operationContext != null) {
LoadEvents.LoadTableMergePartitionEvent loadTableMergePartitionEvent =
new LoadEvents.LoadTableMergePartitionEvent(segmentPath);
@@ -138,22 +139,78 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
String segmentsToBeDeleted =
context.getConfiguration().get(CarbonTableOutputFormat.SEGMENTS_TO_BE_DELETED, "");
List<String> segmentDeleteList = Arrays.asList(segmentsToBeDeleted.split(","));
+ Set<String> segmentSet = new HashSet<>(
+ new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier())
+ .getValidAndInvalidSegments().getValidSegments());
if (updateTime != null) {
- Set<String> segmentSet = new HashSet<>(
- new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier())
- .getValidAndInvalidSegments().getValidSegments());
CarbonUpdateUtil.updateTableMetadataStatus(
segmentSet,
carbonTable,
updateTime,
true,
segmentDeleteList);
+ } else if (uniqueId != null) {
+ // Update the loadstatus with update time to clear cache from driver.
+ CarbonUpdateUtil.updateTableMetadataStatus(
+ segmentSet,
+ carbonTable,
+ uniqueId,
+ true,
+ new ArrayList<String>());
}
} else {
CarbonLoaderUtil.updateTableStatusForFailure(loadModel);
}
}
+ /**
+ * Overwrite the partitions in case of overwrite query. It just updates the partition map files
+ * of all segment files.
+ *
+ * @param loadModel
+ * @return
+ * @throws IOException
+ */
+ private String overwritePartitions(CarbonLoadModel loadModel) throws IOException {
+ CarbonTable table = loadModel.getCarbonDataLoadSchema().getCarbonTable();
+ String currentSegmentPath =
+ CarbonTablePath.getSegmentPath(loadModel.getTablePath(), loadModel.getSegmentId());
+ PartitionMapFileStore partitionMapFileStore = new PartitionMapFileStore();
+ partitionMapFileStore.readAllPartitionsOfSegment(currentSegmentPath);
+ List<List<String>> partitionsToDrop =
+ new ArrayList<List<String>>(partitionMapFileStore.getPartitionMap().values());
+ if (partitionsToDrop.size() > 0) {
+ List<String> validSegments =
+ new SegmentStatusManager(table.getAbsoluteTableIdentifier()).getValidAndInvalidSegments()
+ .getValidSegments();
+ String uniqueId = String.valueOf(System.currentTimeMillis());
+ try {
+ // First drop the partitions from partition mapper files of each segment
+ for (String segment : validSegments) {
+ new PartitionMapFileStore()
+ .dropPartitions(CarbonTablePath.getSegmentPath(table.getTablePath(), segment),
+ new ArrayList<List<String>>(partitionsToDrop), uniqueId, false);
+
+ }
+ } catch (Exception e) {
+ // roll back the drop partitions from carbon store
+ for (String segment : validSegments) {
+ new PartitionMapFileStore()
+ .commitPartitions(CarbonTablePath.getSegmentPath(table.getTablePath(), segment),
+ uniqueId, false, table.getTablePath(), partitionsToDrop.get(0));
+ }
+ }
+ // Commit the removed partitions in carbon store.
+ for (String segment : validSegments) {
+ new PartitionMapFileStore()
+ .commitPartitions(CarbonTablePath.getSegmentPath(table.getTablePath(), segment),
+ uniqueId, true, table.getTablePath(), partitionsToDrop.get(0));
+ }
+ return uniqueId;
+ }
+ return null;
+ }
+
private Object getOperationContext() {
// when validate segments is disabled in thread local update it to CarbonTableInputFormat
CarbonSessionInfo carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index 47c8da9..440720e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -33,10 +33,10 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
+import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
import org.apache.carbondata.processing.loading.DataLoadExecutor;
import org.apache.carbondata.processing.loading.TableProcessingOperations;
-import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable;
import org.apache.carbondata.processing.loading.iterator.CarbonOutputIteratorWrapper;
import org.apache.carbondata.processing.loading.model.CarbonDataLoadSchema;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
@@ -57,7 +57,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
* It also generate and writes dictionary data during load only if dictionary server is configured.
*/
// TODO Move dictionary generater which is coded in spark to MR framework.
-public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, StringArrayWritable> {
+public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, ObjectArrayWritable> {
private static final String LOAD_MODEL = "mapreduce.carbontable.load.model";
private static final String DATABASE_NAME = "mapreduce.carbontable.databaseName";
@@ -228,7 +228,7 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Stri
}
@Override
- public RecordWriter<NullWritable, StringArrayWritable> getRecordWriter(
+ public RecordWriter<NullWritable, ObjectArrayWritable> getRecordWriter(
TaskAttemptContext taskAttemptContext) throws IOException {
final CarbonLoadModel loadModel = getLoadModel(taskAttemptContext.getConfiguration());
loadModel.setTaskNo(taskAttemptContext.getConfiguration().get(
@@ -372,7 +372,7 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Stri
model.setCsvHeaderColumns(columns);
}
- public static class CarbonRecordWriter extends RecordWriter<NullWritable, StringArrayWritable> {
+ public static class CarbonRecordWriter extends RecordWriter<NullWritable, ObjectArrayWritable> {
private CarbonOutputIteratorWrapper iteratorWrapper;
@@ -394,9 +394,9 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Stri
this.future = future;
}
- @Override public void write(NullWritable aVoid, StringArrayWritable strings)
+ @Override public void write(NullWritable aVoid, ObjectArrayWritable objects)
throws InterruptedException {
- iteratorWrapper.write(strings.get());
+ iteratorWrapper.write(objects.get());
}
@Override public void close(TaskAttemptContext taskAttemptContext) throws InterruptedException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/ObjectArrayWritable.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/ObjectArrayWritable.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/ObjectArrayWritable.java
new file mode 100644
index 0000000..d89e7d9
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/ObjectArrayWritable.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.hadoop.internal;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * A Object sequence that is usable as a key or value.
+ */
+public class ObjectArrayWritable implements Writable {
+ private Object[] values;
+
+ public void set(Object[] values) {
+ this.values = values;
+ }
+
+ public Object[] get() {
+ return values;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ int length = in.readInt();
+ values = new Object[length];
+ for (int i = 0; i < length; i++) {
+ byte[] b = new byte[in.readInt()];
+ in.readFully(b);
+ values[i] = new String(b, Charset.defaultCharset());
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(values.length); // write values
+ for (int i = 0; i < values.length; i++) {
+ byte[] b = values[i].toString().getBytes(Charset.defaultCharset());
+ out.writeInt(b.length);
+ out.write(b);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return Arrays.toString(values);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java
index cb43b79..653a49e 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat;
+import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
import org.apache.carbondata.hadoop.test.util.StoreCreator;
import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable;
@@ -86,11 +87,13 @@ public class CarbonTableOutputFormatTest {
.addProperty(CarbonCommonConstants.ENABLE_QUERY_STATISTICS, "false");
}
- public static class Map extends Mapper<NullWritable, StringArrayWritable, NullWritable, StringArrayWritable> {
+ public static class Map extends Mapper<NullWritable, StringArrayWritable, NullWritable, ObjectArrayWritable> {
+ private ObjectArrayWritable writable = new ObjectArrayWritable();
@Override protected void map(NullWritable key, StringArrayWritable value, Context context)
throws IOException, InterruptedException {
- context.write(key, value);
+ writable.set(value.get());
+ context.write(key, writable);
}
}
@@ -100,7 +103,7 @@ public class CarbonTableOutputFormatTest {
Job job = Job.getInstance(configuration);
job.setJarByClass(CarbonTableOutputFormatTest.class);
job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(StringArrayWritable.class);
+ job.setOutputValueClass(ObjectArrayWritable.class);
job.setMapperClass(Map.class);
job.setNumReduceTasks(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 7c288b3..51df525 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -30,7 +30,8 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
SparkUtil4Test.createTaskMockUp(sqlContext)
dropTable
-
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
@@ -234,6 +235,9 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
test("unsorted table creation, query data loading with heap and safe sort config") {
try {
+ sql("drop table if exists origintable1")
+ sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("false", "false", "false")
sql("CREATE TABLE unsortedtable_heap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
@@ -246,6 +250,9 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
test("unsorted table creation, query and data loading with heap and unsafe sort config") {
try {
+ sql("drop table if exists origintable1")
+ sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("false", "true", "false")
sql("CREATE TABLE unsortedtable_heap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
@@ -258,6 +265,9 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
test("unsorted table creation, query and loading with heap and inmemory sort config") {
try {
+ sql("drop table if exists origintable1")
+ sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("false", "false", "true")
sql("CREATE TABLE unsortedtable_heap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
@@ -270,6 +280,9 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
test("unsorted table creation, query and data loading with offheap and safe sort config") {
try {
+ sql("drop table if exists origintable1")
+ sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("true", "false", "false")
sql("CREATE TABLE unsortedtable_offheap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
@@ -282,6 +295,9 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
test("unsorted table creation, query and data loading with offheap and unsafe sort config") {
try {
+ sql("drop table if exists origintable1")
+ sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("true", "true", "false")
sql("CREATE TABLE unsortedtable_offheap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
@@ -294,6 +310,9 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
test("unsorted table creation, query and data loading with offheap and inmemory sort config") {
try {
+ sql("drop table if exists origintable1")
+ sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("true", "false", "true")
sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala
index e44ccd6..2e2c1f0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionBadRecordLoggerTest.scala
@@ -237,6 +237,7 @@ class StandardPartitionBadRecordLoggerTest extends QueryTest with BeforeAndAfter
override def afterAll {
drop()
CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
new file mode 100644
index 0000000..0dbf1e4
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
@@ -0,0 +1,684 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.standardpartition
+
+import java.util
+import java.util.concurrent.{Callable, ExecutorService, Executors}
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, Ignore}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterAll {
+ var executorService: ExecutorService = _
+ override def beforeAll {
+ dropTable
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ sql(
+ """
+ | CREATE TABLE originTable (empno int, empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ }
+
+ def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int): Unit = {
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
+ val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
+ carbonTable.getTablePath)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
+ val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ override def accept(file: CarbonFile): Boolean = {
+ return file.getName.endsWith(".partitionmap")
+ }
+ })
+ assert(dataFiles.length == partitions)
+ }
+
+ test("data loading for global sort partition table for one partition column") {
+ sql(
+ """
+ | CREATE TABLE partitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"', 'GLOBAL_SORT_PARTITIONS'='1')""")
+
+ validateDataFiles("default_partitionone", "0", 1)
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionone order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("data loading for global partition table for two partition column") {
+ sql(
+ """
+ | CREATE TABLE partitiontwo (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (doj Timestamp, empname String)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiontwo OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ validateDataFiles("default_partitiontwo", "0", 1)
+
+ checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitiontwo order by empno"),
+ sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
+
+ }
+
+ test("data loading for global sort partition table for one static partition column") {
+ sql(
+ """
+ | CREATE TABLE staticpartitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""insert into staticpartitionone PARTITION(empno='1') select empname,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary from originTable""")
+
+ validateDataFiles("default_staticpartitionone", "0", 1)
+ }
+
+ test("single pass loading for global sort partition table for one partition column") {
+ sql(
+ """
+ | CREATE TABLE singlepasspartitionone (empname String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (designation String)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE singlepasspartitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"', 'SINGLE_PASS'='true')""")
+
+ validateDataFiles("default_singlepasspartitionone", "0", 1)
+ }
+
+ test("data loading for global sort partition table for one static partition column with load syntax") {
+ sql(
+ """
+ | CREATE TABLE loadstaticpartitionone (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadstaticpartitionone PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql("select distinct empno from loadstaticpartitionone"), Seq(Row(1)))
+ }
+
+ test("overwrite global sort partition table for one static partition column with load syntax") {
+ sql(
+ """
+ | CREATE TABLE loadstaticpartitiononeoverwrite (empname String, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadstaticpartitiononeoverwrite PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val rows = sql("select count(*) from loadstaticpartitiononeoverwrite").collect()
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadstaticpartitiononeoverwrite PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadstaticpartitiononeoverwrite PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE loadstaticpartitiononeoverwrite PARTITION(empno='1') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql("select count(*) from loadstaticpartitiononeoverwrite"), rows)
+ }
+
+ test("test global sort partition column with special characters") {
+ sql(
+ """
+ | CREATE TABLE loadpartitionwithspecialchar (empno int, designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empname String)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data_with_special_char.csv' INTO TABLE loadpartitionwithspecialchar OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql("select count(*) from loadpartitionwithspecialchar"), Seq(Row(10)))
+ checkAnswer(sql("select count(*) from loadpartitionwithspecialchar where empname='sibi=56'"), Seq(Row(1)))
+ checkAnswer(sql("select count(*) from loadpartitionwithspecialchar where empname='arvind,ss'"), Seq(Row(1)))
+ }
+
+ test("concurrent global sort partition table load test") {
+ executorService = Executors.newCachedThreadPool()
+ sql(
+ """
+ | CREATE TABLE partitionmultiplethreeconcurrent (empno int, doj Timestamp,
+ | workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (workgroupcategory int, empname String, designation String)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('DICTIONARY_INCLUDE'='empname,designation,deptname', 'SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+
+ val tasks = new util.ArrayList[Callable[String]]()
+ tasks.add(new QueryTask(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethreeconcurrent OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')"""))
+ tasks.add(new QueryTask(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethreeconcurrent OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')"""))
+ tasks.add(new QueryTask(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethreeconcurrent OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')"""))
+ val results = executorService.invokeAll(tasks)
+ for (i <- 0 until tasks.size()) {
+ val res = results.get(i).get
+ assert("PASS".equals(res))
+ }
+ executorService.shutdown()
+ checkAnswer(sql("select count(*) from partitionmultiplethreeconcurrent"), Seq(Row(30)))
+ }
+
+ class QueryTask(query: String) extends Callable[String] {
+ override def call(): String = {
+ var result = "PASS"
+ try {
+ LOGGER.info("Executing :" + Thread.currentThread().getName)
+ sql(query)
+ } catch {
+ case ex: Exception =>
+ ex.printStackTrace()
+ result = "FAIL"
+ }
+ result
+ }
+ }
+
+ test("global sort bad record test with null values") {
+ sql(s"""CREATE TABLE IF NOT EXISTS emp1 (emp_no int,ename string,job string,mgr_id int,date_of_joining string,salary int,bonus int) partitioned by (dept_no int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/emp.csv' overwrite INTO TABLE emp1 OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '\')""")
+ val rows = sql(s"select count(*) from emp1").collect()
+ sql(s"""LOAD DATA INPATH '$resourcesPath/emp.csv' overwrite INTO TABLE emp1 OPTIONS('DELIMITER'=',', 'QUOTECHAR'= '\','BAD_RECORDS_ACTION'='FORCE')""")
+ checkAnswer(sql(s"select count(*) from emp1"), rows)
+ }
+
+ test("global sort badrecords on partition column") {
+ sql("create table badrecordsPartition(intField1 int, stringField1 string) partitioned by (intField2 int) stored by 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')")
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsPartition options('bad_records_action'='force')")
+ sql("select count(*) from badrecordsPartition").show()
+ checkAnswer(sql("select count(*) cnt from badrecordsPartition where intfield2 is null"), Seq(Row(9)))
+ checkAnswer(sql("select count(*) cnt from badrecordsPartition where intfield2 is not null"), Seq(Row(2)))
+ }
+
+ test("global sort badrecords fail on partition column") {
+ sql("create table badrecordsPartitionfail(intField1 int, stringField1 string) partitioned by (intField2 int) stored by 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')")
+ intercept[Exception] {
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsPartitionfail options('bad_records_action'='fail')")
+
+ }
+ }
+
+ test("global sort badrecords ignore on partition column") {
+ sql("create table badrecordsPartitionignore(intField1 int, stringField1 string) partitioned by (intField2 int) stored by 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')")
+ sql("create table badrecordsignore(intField1 int,intField2 int, stringField1 string) stored by 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')")
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsPartitionignore options('bad_records_action'='ignore')")
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsignore options('bad_records_action'='ignore')")
+ checkAnswer(sql("select count(*) cnt from badrecordsPartitionignore where intfield2 is null"), sql("select count(*) cnt from badrecordsignore where intfield2 is null"))
+ checkAnswer(sql("select count(*) cnt from badrecordsPartitionignore where intfield2 is not null"), sql("select count(*) cnt from badrecordsignore where intfield2 is not null"))
+ }
+
+
+ test("global sort test partition fails on int null partition") {
+ sql("create table badrecordsPartitionintnull(intField1 int, stringField1 string) partitioned by (intField2 int) stored by 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')")
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsPartitionintnull options('bad_records_action'='force')")
+ checkAnswer(sql("select count(*) cnt from badrecordsPartitionintnull where intfield2 = 13"), Seq(Row(1)))
+ }
+
+ test("global sort test partition fails on int null partition read alternate") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT, "false")
+ sql("create table badrecordsPartitionintnullalt(intField1 int, stringField1 string) partitioned by (intField2 int) stored by 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')")
+ sql(s"load data local inpath '$resourcesPath/data_partition_badrecords.csv' into table badrecordsPartitionintnullalt options('bad_records_action'='force')")
+ checkAnswer(sql("select count(*) cnt from badrecordsPartitionintnullalt where intfield2 = 13"), Seq(Row(1)))
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT, CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT_DEFAULT)
+ }
+
+ test("global sort static column partition with load command") {
+ sql(
+ """
+ | CREATE TABLE staticpartitionload (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int,projectenddate Date,doj Timestamp)
+ | PARTITIONED BY (empname String)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE staticpartitionload partition(empname='ravi') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val frame = sql("select empno,empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from staticpartitionload")
+ checkExistence(sql(s"""SHOW PARTITIONS staticpartitionload"""), true, "empname=ravi")
+ }
+
+ test("overwriting global sort static partition table for date partition column on insert query") {
+ sql(
+ """
+ | CREATE TABLE staticpartitiondateinsert (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int)
+ | PARTITIONED BY (projectenddate Date,doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert overwrite table staticpartitiondateinsert PARTITION(projectenddate='2016-06-29',doj='2010-12-29 00:00:00') select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary from originTable where projectenddate=cast('2016-06-29' as Date)""")
+ // sql(s"""insert overwrite table partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ checkAnswer(sql("select * from staticpartitiondateinsert where projectenddate=cast('2016-06-29' as Date)"),
+ sql("select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable where projectenddate=cast('2016-06-29' as Date)"))
+ }
+
+
+ test("dynamic and static global sort partition table with load syntax") {
+ sql(
+ """
+ | CREATE TABLE loadstaticpartitiondynamic (designation String, doj Timestamp,
+ | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String,
+ | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (empno int, empname String)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE loadstaticpartitiondynamic PARTITION(empno='1', empname) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ checkAnswer(sql(s"select count(*) from loadstaticpartitiondynamic where empno=1"), sql(s"select count(*) from loadstaticpartitiondynamic"))
+ }
+
+ test("dynamic and static global sort partition table with overwrite ") {
+ sql(
+ """
+ | CREATE TABLE insertstaticpartitiondynamic (designation String, doj Timestamp,salary int)
+ | PARTITIONED BY (empno int, empname String)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE insertstaticpartitiondynamic PARTITION(empno, empname) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val rows = sql(s"select count(*) from insertstaticpartitiondynamic").collect()
+ sql("""insert overwrite table insertstaticpartitiondynamic PARTITION(empno='1', empname) select designation, doj, salary, empname from insertstaticpartitiondynamic""")
+
+ checkAnswer(sql(s"select count(*) from insertstaticpartitiondynamic where empno=1"), rows)
+
+ intercept[Exception] {
+ sql("""insert overwrite table insertstaticpartitiondynamic PARTITION(empno, empname='ravi') select designation, doj, salary, empname from insertstaticpartitiondynamic""")
+ }
+
+ }
+
+ test("overwriting global sort all partition on table and do compaction") {
+ sql(
+ """
+ | CREATE TABLE partitionallcompaction (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (deptname String,doj Timestamp,projectcode int)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE partitionallcompaction OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE partitionallcompaction PARTITION(deptname='Learning', doj, projectcode) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"') """)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE partitionallcompaction PARTITION(deptname='configManagement', doj, projectcode) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE partitionallcompaction PARTITION(deptname='network', doj, projectcode) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE partitionallcompaction PARTITION(deptname='protocol', doj, projectcode) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE partitionallcompaction PARTITION(deptname='security', doj, projectcode) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql("ALTER TABLE partitionallcompaction COMPACT 'MAJOR'").collect()
+ checkExistence(sql(s"""SHOW segments for table partitionallcompaction"""), true, "Marked for Delete")
+ }
+
+ test("Test global sort overwrite static partition ") {
+ sql(
+ """
+ | CREATE TABLE weather6 (type String)
+ | PARTITIONED BY (year int, month int, day int)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+
+ sql("insert into weather6 partition(year=2014, month=5, day=25) select 'rainy'")
+ sql("insert into weather6 partition(year=2014, month=4, day=23) select 'cloudy'")
+ sql("insert overwrite table weather6 partition(year=2014, month=5, day=25) select 'sunny'")
+ checkExistence(sql("select * from weather6"), true, "sunny")
+ checkAnswer(sql("select count(*) from weather6"), Seq(Row(2)))
+ }
+
+ test("Test global sort overwrite static partition with wrong int value") {
+ sql(
+ """
+ | CREATE TABLE weather7 (type String)
+ | PARTITIONED BY (year int, month int, day int)
+ | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')
+ """.stripMargin)
+
+ sql("insert into weather7 partition(year=2014, month=05, day=25) select 'rainy'")
+ sql("insert into weather7 partition(year=2014, month=04, day=23) select 'cloudy'")
+ sql("insert overwrite table weather7 partition(year=2014, month=05, day=25) select 'sunny'")
+ checkExistence(sql("select * from weather7"), true, "sunny")
+ checkAnswer(sql("select count(*) from weather7"), Seq(Row(2)))
+ sql("insert into weather7 partition(year=2014, month, day) select 'rainy1',06,25")
+ sql("insert into weather7 partition(year=2014, month=01, day) select 'rainy2',27")
+ sql("insert into weather7 partition(year=2014, month=01, day=02) select 'rainy3'")
+ checkAnswer(sql("select count(*) from weather7 where month=1"), Seq(Row(2)))
+ }
+
+
+ test("test overwrite missed scenarios") {
+ sql(s"""create table carbon_test(
+ id string,
+ name string
+ )
+ PARTITIONED BY(record_date int)
+ STORED BY 'org.apache.carbondata.format'
+ TBLPROPERTIES('SORT_COLUMNS'='id')""")
+ sql(s"""create table carbon_test_hive(
+ id string,
+ name string
+ )
+ PARTITIONED BY(record_date int)""")
+ sql(s"""set hive.exec.dynamic.partition.mode=nonstrict""")
+ sql(s"""insert overwrite table carbon_test partition(record_date) select '1','kim',unix_timestamp('2018-02-05','yyyy-MM-dd') as record_date""")
+ sql(s"""insert overwrite table carbon_test_hive partition(record_date) select '1','kim',unix_timestamp('2018-02-05','yyyy-MM-dd') as record_date""")
+
+ checkAnswer(sql(s"""select * from carbon_test where record_date=1517817600"""), sql(s"""select * from carbon_test_hive where record_date=1517817600"""))
+ sql(s"""insert overwrite table carbon_test partition(record_date) select '1','kim1',unix_timestamp('2018-02-06','yyyy-MM-dd') as record_date """)
+ sql(s"""insert overwrite table carbon_test_hive partition(record_date) select '1','kim1',unix_timestamp('2018-02-06','yyyy-MM-dd') as record_date """)
+
+ checkAnswer(sql(s"""select * from carbon_test where record_date=1517817600"""), sql(s"""select * from carbon_test_hive where record_date=1517817600"""))
+ checkAnswer(sql(s"""select * from carbon_test where record_date=1517904000"""), sql(s"""select * from carbon_test_hive where record_date=1517904000"""))
+ sql(s"""insert overwrite table carbon_test partition(record_date) select '1','kim2',unix_timestamp('2018-02-07','yyyy-MM-dd') as record_date""")
+ sql(s"""insert overwrite table carbon_test_hive partition(record_date) select '1','kim2',unix_timestamp('2018-02-07','yyyy-MM-dd') as record_date""")
+
+ checkAnswer(sql(s"""select * from carbon_test where record_date=1517817600"""), sql(s"""select * from carbon_test_hive where record_date=1517817600"""))
+ checkAnswer(sql(s"""select * from carbon_test where record_date=1517904000"""), sql(s"""select * from carbon_test_hive where record_date=1517904000"""))
+ checkAnswer(sql(s"""select * from carbon_test where record_date=1517990400"""), sql(s"""select * from carbon_test_hive where record_date=1517990400"""))
+ }
+
+ test("test overwrite with timestamp partition column") {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+ sql("DROP TABLE IF EXISTS origintable")
+ sql(
+ """
+ | CREATE TABLE origintable
+ | (id Int,
+ | vin String,
+ | logdate Timestamp,
+ | phonenumber Long,
+ | country String,
+ | area String,
+ | salary Int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/partition_data.csv' into table origintable
+ """)
+
+ sql("DROP TABLE IF EXISTS partitiontable0")
+ sql("DROP TABLE IF EXISTS partitiontable0_hive")
+ sql(
+ """
+ | CREATE TABLE partitiontable0
+ | (id Int,
+ | vin String,
+ | phonenumber Long,
+ | country String,
+ | area String,
+ | salary Int)
+ | PARTITIONED BY (logdate Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='id,vin')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE partitiontable0_hive
+ | (id Int,
+ | vin String,
+ | phonenumber Long,
+ | country String,
+ | area String,
+ | salary Int)
+ | PARTITIONED BY (logdate Timestamp)
+ """.stripMargin)
+ sql(s"""set hive.exec.dynamic.partition.mode=nonstrict""")
+
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/partition_data.csv' into table partitiontable0
+ """)
+
+ sql(
+ s"""
+ insert into partitiontable0_hive select * from partitiontable0
+ """)
+
+ checkAnswer(sql(
+ s"""
+ | SELECT logdate,id,vin,phonenumber,country,area,salary
+ | FROM partitiontable0 where logdate = '2016-02-12'
+ """.stripMargin), sql(
+ s"""
+ | SELECT logdate,id,vin,phonenumber,country,area,salary
+ | FROM partitiontable0_hive where logdate = '2016-02-12'
+ """.stripMargin))
+
+ sql("insert into table partitiontable0 partition(logdate='2018-02-15 00:00:00') " +
+ "select id,vin,phonenumber,country,area,salary from origintable")
+ sql("insert into table partitiontable0_hive partition(logdate='2018-02-15 00:00:00') " +
+ "select id,vin,phonenumber,country,area,salary from origintable")
+ checkAnswer(sql(
+ s"""
+ | SELECT logdate,id,vin,phonenumber,country,area,salary
+ | FROM partitiontable0 where logdate = '2018-02-15'
+ """.stripMargin), sql(
+ s"""
+ | SELECT logdate,id,vin,phonenumber,country,area,salary
+ | FROM partitiontable0_hive where logdate = '2018-02-15'
+ """.stripMargin))
+
+ checkAnswer(sql(
+ s"""
+ | SELECT count(*) FROM partitiontable0""".stripMargin), sql(
+ s"""
+ | SELECT count(*) FROM partitiontable0_hive""".stripMargin))
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ }
+
+ test("test overwrite with date partition column") {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ sql("DROP TABLE IF EXISTS origintable")
+ sql(
+ """
+ | CREATE TABLE origintable
+ | (id Int,
+ | vin String,
+ | logdate date,
+ | phonenumber Long,
+ | country String,
+ | area String,
+ | salary Int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/partition_data.csv' into table origintable
+ """)
+
+ sql("DROP TABLE IF EXISTS partitiontable0")
+ sql("DROP TABLE IF EXISTS partitiontable0_hive")
+ sql(
+ """
+ | CREATE TABLE partitiontable0
+ | (id Int,
+ | vin String,
+ | phonenumber Long,
+ | country String,
+ | area String,
+ | salary Int)
+ | PARTITIONED BY (logdate date)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='id,vin')
+ """.stripMargin)
+ sql(
+ """
+ | CREATE TABLE partitiontable0_hive
+ | (id Int,
+ | vin String,
+ | phonenumber Long,
+ | country String,
+ | area String,
+ | salary Int)
+ | PARTITIONED BY (logdate date)
+ """.stripMargin)
+ sql(s"""set hive.exec.dynamic.partition.mode=nonstrict""")
+
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/partition_data.csv' into table partitiontable0
+ """)
+
+ sql(
+ s"""
+ insert into partitiontable0_hive select * from partitiontable0
+ """)
+
+ checkAnswer(sql(
+ s"""
+ | SELECT logdate,id,vin,phonenumber,country,area,salary
+ | FROM partitiontable0 where logdate = '2016-02-12'
+ """.stripMargin), sql(
+ s"""
+ | SELECT logdate,id,vin,phonenumber,country,area,salary
+ | FROM partitiontable0_hive where logdate = '2016-02-12'
+ """.stripMargin))
+
+ sql("insert into table partitiontable0 partition(logdate='2018-02-15') " +
+ "select id,vin,phonenumber,country,area,salary from origintable")
+ sql("insert into table partitiontable0_hive partition(logdate='2018-02-15') " +
+ "select id,vin,phonenumber,country,area,salary from origintable")
+ checkAnswer(sql(
+ s"""
+ | SELECT logdate,id,vin,phonenumber,country,area,salary
+ | FROM partitiontable0 where logdate = '2018-02-15'
+ """.stripMargin), sql(
+ s"""
+ | SELECT logdate,id,vin,phonenumber,country,area,salary
+ | FROM partitiontable0_hive where logdate = '2018-02-15'
+ """.stripMargin))
+
+ checkAnswer(sql(
+ s"""
+ | SELECT count(*) FROM partitiontable0""".stripMargin), sql(
+ s"""
+ | SELECT count(*) FROM partitiontable0_hive""".stripMargin))
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy")
+ }
+
+
+
+ override def afterAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION ,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT)
+// dropTable
+ if (executorService != null && !executorService.isShutdown) {
+ executorService.shutdownNow()
+ }
+ }
+
+ def dropTable = {
+ sql("drop table if exists originTable")
+ sql("drop table if exists originMultiLoads")
+ sql("drop table if exists partitionone")
+ sql("drop table if exists partitiontwo")
+ sql("drop table if exists partitionthree")
+ sql("drop table if exists partitionmultiplethree")
+ sql("drop table if exists insertpartitionthree")
+ sql("drop table if exists staticpartitionone")
+ sql("drop table if exists singlepasspartitionone")
+ sql("drop table if exists loadstaticpartitionone")
+ sql("drop table if exists loadstaticpartitiononeoverwrite")
+ sql("drop table if exists streamingpartitionedtable")
+ sql("drop table if exists mergeindexpartitionthree")
+ sql("drop table if exists loadstaticpartitiononeissue")
+ sql("drop table if exists partitionmultiplethreeconcurrent")
+ sql("drop table if exists loadpartitionwithspecialchar")
+ sql("drop table if exists emp1")
+ sql("drop table if exists restorepartition")
+ sql("drop table if exists casesensitivepartition")
+ sql("drop table if exists badrecordsPartition")
+ sql("drop table if exists staticpartitionload")
+ sql("drop table if exists badrecordsPartitionignore")
+ sql("drop table if exists badrecordsPartitionfail")
+ sql("drop table if exists badrecordsignore")
+ sql("drop table if exists badrecordsPartitionintnull")
+ sql("drop table if exists badrecordsPartitionintnullalt")
+ sql("drop table if exists partitiondateinsert")
+ sql("drop table if exists staticpartitiondateinsert")
+ sql("drop table if exists loadstaticpartitiondynamic")
+ sql("drop table if exists insertstaticpartitiondynamic")
+ sql("drop table if exists partitionallcompaction")
+ sql("drop table if exists weather6")
+ sql("drop table if exists weather7")
+ sql("drop table if exists uniqdata_hive_static")
+ sql("drop table if exists uniqdata_hive_dynamic")
+ sql("drop table if exists uniqdata_string_static")
+ sql("drop table if exists uniqdata_string_dynamic")
+ sql("drop table if exists partitionLoadTable")
+ sql("drop table if exists noLoadTable")
+ sql("drop table if exists carbon_test")
+ sql("drop table if exists carbon_test_hive")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 669d6e7..10da906 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -506,6 +506,9 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
override def afterAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION ,
CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT)
dropTable
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
index 8d31134..841185b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
@@ -77,13 +77,30 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
| PARTITIONED BY (projectenddate Date,doj Timestamp)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
+ sql(
+ """
+ | CREATE TABLE partitiondateinserthive (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int)
+ | PARTITIONED BY (projectenddate Date,doj Timestamp)
+ """.stripMargin)
+ sql(s"""set hive.exec.dynamic.partition.mode=nonstrict""")
sql(s"""insert into partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
sql(s"""insert into partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
sql(s"""insert into partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
sql(s"""insert into partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
sql(s"""insert overwrite table partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable where projectenddate=cast('2016-06-29' as Date)""")
+
+ sql(s"""insert into partitiondateinserthive select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into partitiondateinserthive select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into partitiondateinserthive select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert into partitiondateinserthive select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
+ sql(s"""insert overwrite table partitiondateinserthive select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable where projectenddate=cast('2016-06-29' as Date)""")
+
checkAnswer(sql("select * from partitiondateinsert"),
- sql("select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable where projectenddate=cast('2016-06-29' as Date)"))
+ sql("select * from partitiondateinserthive"))
}
test("dynamic and static partition table with load syntax") {
@@ -182,7 +199,7 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
sql("insert overwrite table uniqdata_string_dynamic partition(active_emui_version='xxx') select CUST_ID, CUST_NAME,DOB,doj, bigint_column1, bigint_column2, decimal_column1, decimal_column2,double_column1, double_column2,integer_column1 from uniqdata_hive_dynamic limit 10")
assert(sql("select * from uniqdata_string_dynamic").collect().length == 2)
sql("insert overwrite table uniqdata_string_dynamic select CUST_ID, CUST_NAME,DOB,doj, bigint_column1, bigint_column2, decimal_column1, decimal_column2,double_column1, double_column2,integer_column1,ACTIVE_EMUI_VERSION from uniqdata_hive_dynamic limit 10")
- checkAnswer(sql("select * from uniqdata_string_dynamic"), sql("select * from uniqdata_hive_dynamic"))
+ assert(sql("select * from uniqdata_string_dynamic").collect().length == 2)
}
test("test insert overwrite on static partition") {
@@ -193,7 +210,7 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
sql("insert overwrite table uniqdata_string_static partition(active_emui_version='xxx') select CUST_ID, CUST_NAME,DOB,doj, bigint_column1, bigint_column2, decimal_column1, decimal_column2,double_column1, double_column2,integer_column1 from uniqdata_hive_static limit 10")
assert(sql("select * from uniqdata_string_static").collect().length == 2)
sql("insert overwrite table uniqdata_string_static select CUST_ID, CUST_NAME,DOB,doj, bigint_column1, bigint_column2, decimal_column1, decimal_column2,double_column1, double_column2,integer_column1,active_emui_version from uniqdata_hive_static limit 10")
- checkAnswer(sql("select * from uniqdata_string_static"), sql("select * from uniqdata_hive_static"))
+ assert(sql("select * from uniqdata_string_static").collect().length == 2)
}
test("overwrite whole partition table with empty data") {
@@ -202,7 +219,7 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
sql("insert into partitionLoadTable select 'abd',5,'xyz'")
sql("create table noLoadTable (name string, age int, address string) stored by 'carbondata'")
sql("insert overwrite table partitionLoadTable select * from noLoadTable")
- checkAnswer(sql("select * from partitionLoadTable"), sql("select * from noLoadTable"))
+ assert(sql("select * from partitionLoadTable").collect().length == 2)
}
override def afterAll = {
@@ -224,6 +241,7 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
sql("drop table if exists uniqdata_string_dynamic")
sql("drop table if exists partitionLoadTable")
sql("drop table if exists noLoadTable")
+ sql("drop table if exists partitiondateinserthive")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
index 8be70a9..30e4fc9 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
@@ -95,7 +95,8 @@ object DataLoadProcessBuilderOnSpark {
}
}
- var numPartitions = CarbonDataProcessorUtil.getGlobalSortPartitions(configuration)
+ var numPartitions = CarbonDataProcessorUtil.getGlobalSortPartitions(
+ configuration.getDataLoadProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS))
if (numPartitions <= 0) {
numPartitions = convertRDD.partitions.length
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 21de003..4b7d3f7 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -95,15 +95,69 @@ object DataLoadProcessorStepOnSpark {
}
}
+ def inputAndconvertFunc(
+ rows: Iterator[Array[AnyRef]],
+ index: Int,
+ modelBroadcast: Broadcast[CarbonLoadModel],
+ partialSuccessAccum: Accumulator[Int],
+ rowCounter: Accumulator[Int],
+ keepActualData: Boolean = false): Iterator[CarbonRow] = {
+ val model: CarbonLoadModel = modelBroadcast.value.getCopyWithTaskNo(index.toString)
+ val conf = DataLoadProcessBuilder.createConfiguration(model)
+ val rowParser = new RowParserImpl(conf.getDataFields, conf)
+ val isRawDataRequired = CarbonDataProcessorUtil.isRawDataRequired(conf)
+ val badRecordLogger = BadRecordsLoggerProvider.createBadRecordLogger(conf)
+ if (keepActualData) {
+ conf.getDataFields.foreach(_.setUseActualData(keepActualData))
+ }
+ val rowConverter = new RowConverterImpl(conf.getDataFields, conf, badRecordLogger)
+ rowConverter.initialize()
+
+ TaskContext.get().addTaskCompletionListener { context =>
+ val hasBadRecord: Boolean = CarbonBadRecordUtil.hasBadRecord(model)
+ close(conf, badRecordLogger, rowConverter)
+ GlobalSortHelper.badRecordsLogger(model, partialSuccessAccum, hasBadRecord)
+ }
+
+ TaskContext.get().addTaskFailureListener { (t: TaskContext, e: Throwable) =>
+ val hasBadRecord : Boolean = CarbonBadRecordUtil.hasBadRecord(model)
+ close(conf, badRecordLogger, rowConverter)
+ GlobalSortHelper.badRecordsLogger(model, partialSuccessAccum, hasBadRecord)
+
+ wrapException(e, model)
+ }
+
+ new Iterator[CarbonRow] {
+ override def hasNext: Boolean = rows.hasNext
+
+ override def next(): CarbonRow = {
+ var row : CarbonRow = null
+ if(isRawDataRequired) {
+ val rawRow = rows.next()
+ row = new CarbonRow(rowParser.parseRow(rawRow), rawRow)
+ } else {
+ row = new CarbonRow(rowParser.parseRow(rows.next()))
+ }
+ row = rowConverter.convert(row)
+ rowCounter.add(1)
+ row
+ }
+ }
+ }
+
def convertFunc(
rows: Iterator[CarbonRow],
index: Int,
modelBroadcast: Broadcast[CarbonLoadModel],
partialSuccessAccum: Accumulator[Int],
- rowCounter: Accumulator[Int]): Iterator[CarbonRow] = {
+ rowCounter: Accumulator[Int],
+ keepActualData: Boolean = false): Iterator[CarbonRow] = {
val model: CarbonLoadModel = modelBroadcast.value.getCopyWithTaskNo(index.toString)
val conf = DataLoadProcessBuilder.createConfiguration(model)
val badRecordLogger = BadRecordsLoggerProvider.createBadRecordLogger(conf)
+ if (keepActualData) {
+ conf.getDataFields.foreach(_.setUseActualData(keepActualData))
+ }
val rowConverter = new RowConverterImpl(conf.getDataFields, conf, badRecordLogger)
rowConverter.initialize()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/758d03e7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
index dfda92c..2fdd2b1 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
@@ -48,6 +48,7 @@ object ValidateUtil {
if (sortScope != null) {
// Don't support use global sort on partitioned table.
if (carbonTable.getPartitionInfo(carbonTable.getTableName) != null &&
+ !carbonTable.isHivePartitionTable &&
sortScope.equalsIgnoreCase(SortScopeOptions.SortScope.GLOBAL_SORT.toString)) {
throw new MalformedCarbonCommandException("Don't support use global sort on partitioned " +
"table.")
[13/16] carbondata git commit: [CARBONDATA-2187][PARTITION] Partition
restructure for new folder structure and supporting partition location
feature
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
index 2b0dd09..f238d2b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
@@ -17,13 +17,14 @@
package org.apache.carbondata.spark.testsuite.standardpartition
import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -49,24 +50,19 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
}
- def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int, partitionMapFiles: Int): Unit = {
+ def validateDataFiles(tableUniqueName: String, segmentId: String, partition: Int, indexes: Int): Unit = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
- val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
- val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
- override def accept(file: CarbonFile): Boolean = {
- return file.getName.endsWith(".carbondata")
- }
- })
- assert(dataFiles.length == partitions)
- val partitionFile = carbonFile.listFiles(new CarbonFileFilter() {
- override def accept(file: CarbonFile): Boolean = {
- return file.getName.endsWith(".partitionmap")
- }
- })
- assert(partitionFile.length == partitionMapFiles)
+ val partitions = CarbonFilters
+ .getPartitions(Seq.empty,
+ sqlContext.sparkSession,
+ TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName)))
+ assert(partitions.get.length == partition)
+ val details = SegmentStatusManager.readLoadMetadata(tablePath.getMetadataDirectoryPath)
+ val segLoad = details.find(_.getLoadName.equals(segmentId)).get
+ val seg = new SegmentFileStore(carbonTable.getTablePath, segLoad.getSegmentFile)
+ assert(seg.getIndexFiles.size == indexes)
}
test("clean up partition table for int partition column") {
@@ -89,11 +85,10 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
sql(s"""select count (*) from originTable where empno=11"""))
sql(s"""ALTER TABLE partitionone DROP PARTITION(empno='11')""")
- validateDataFiles("default_partitionone", "0", 10, 2)
+ validateDataFiles("default_partitionone", "0", 9, 9)
sql(s"CLEAN FILES FOR TABLE partitionone").show()
-
+ validateDataFiles("default_partitionone", "0", 9, 9)
checkExistence(sql(s"""SHOW PARTITIONS partitionone"""), false, "empno=11")
- validateDataFiles("default_partitionone", "0", 9, 1)
checkAnswer(
sql(s"""select count (*) from partitionone where empno=11"""),
Seq(Row(0)))
@@ -113,11 +108,11 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmany OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmany OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""ALTER TABLE partitionmany DROP PARTITION(deptname='Learning')""")
- validateDataFiles("default_partitionmany", "0", 10, 2)
- validateDataFiles("default_partitionmany", "1", 10, 2)
+ validateDataFiles("default_partitionmany", "0", 8, 8)
+ validateDataFiles("default_partitionmany", "1", 8, 8)
sql(s"CLEAN FILES FOR TABLE partitionmany").show()
- validateDataFiles("default_partitionmany", "0", 8, 1)
- validateDataFiles("default_partitionmany", "1", 8, 1)
+ validateDataFiles("default_partitionmany", "0", 8, 8)
+ validateDataFiles("default_partitionmany", "1", 8, 8)
checkExistence(sql(s"""SHOW PARTITIONS partitionmany"""), false, "deptname=Learning", "projectcode=928479")
checkAnswer(
sql(s"""select count (*) from partitionmany where deptname='Learning'"""),
@@ -142,7 +137,7 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='protocol')""")
sql(s"""ALTER TABLE partitionall DROP PARTITION(deptname='security')""")
assert(sql(s"""SHOW PARTITIONS partitionall""").collect().length == 0)
- validateDataFiles("default_partitionall", "0", 10, 6)
+ validateDataFiles("default_partitionall", "0", 0, 0)
sql(s"CLEAN FILES FOR TABLE partitionall").show()
validateDataFiles("default_partitionall", "0", 0, 0)
checkAnswer(
@@ -150,6 +145,30 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
Seq(Row(0)))
}
+ test("clean up after deleting segments on table") {
+ sql(
+ """
+ | CREATE TABLE partitionalldeleteseg (empno int, empname String, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp, projectenddate Date,attendance int,
+ | utilization int,salary int)
+ | PARTITIONED BY (deptname String,doj Timestamp,projectcode int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionalldeleteseg OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionalldeleteseg OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionalldeleteseg OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionalldeleteseg OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ assert(sql(s"show segments for table partitionalldeleteseg").count == 4)
+ checkAnswer(sql(s"Select count(*) from partitionalldeleteseg"), Seq(Row(40)))
+ sql(s"delete from table partitionalldeleteseg where segment.id in (1)").show()
+ checkExistence(sql(s"show segments for table partitionalldeleteseg"), true, "Marked for Delete")
+ checkAnswer(sql(s"Select count(*) from partitionalldeleteseg"), Seq(Row(30)))
+ sql(s"CLEAN FILES FOR TABLE partitionalldeleteseg").show()
+ assert(sql(s"show segments for table partitionalldeleteseg").count == 3)
+ }
+
+
override def afterAll = {
dropTable
}
@@ -162,6 +181,7 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
sql("drop table if exists partitionmany")
sql("drop table if exists partitionshow")
sql("drop table if exists staticpartition")
+ sql("drop table if exists partitionalldeleteseg")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
index 22ebd80..33e761f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
@@ -20,11 +20,7 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.{CarbonMetadata, PartitionMapFileStore}
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonTablePath
class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndAfterAll {
@@ -49,23 +45,6 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
}
- def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int): Unit = {
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
- val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
- val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
- override def accept(file: CarbonFile): Boolean = {
- return CarbonTablePath.isCarbonDataFile(file.getName) ||
- CarbonTablePath.isCarbonIndexFile(file.getName)
- }
- })
- assert(dataFiles.length > 1)
- val pstore = new PartitionMapFileStore()
- pstore.readAllPartitionsOfSegment(segmentDir)
- println(pstore.getPartitionMap)
- }
test("data compaction for partition table for one partition column") {
sql(
@@ -83,9 +62,7 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql("ALTER TABLE partitionone COMPACT 'MINOR'").collect()
-
- validateDataFiles("default_partitionone", "0.1", 1)
-
+ checkExistence(sql("show segments for table partitionone"), true, "0.1")
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionone where empno=11 order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where empno=11 order by empno"))
@@ -107,9 +84,7 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionthree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionthree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql("ALTER TABLE partitionthree COMPACT 'MINOR'").collect()
-
- validateDataFiles("default_partitionthree", "0.1", 1)
-
+ checkExistence(sql("show segments for table partitionthree"), true, "0.1")
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionthree where workgroupcategory=1 and empname='arvind' and designation='SE' order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable where workgroupcategory=1 and empname='arvind' and designation='SE' order by empno"))
}
@@ -129,6 +104,7 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql("ALTER TABLE partitionmajor COMPACT 'MINOR'").collect()
+ checkExistence(sql("show segments for table partitionmajor"), true, "0.1")
sql(s"""ALTER TABLE partitionmajor DROP PARTITION(workgroupcategory='1')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
@@ -136,7 +112,7 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmajor OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
val rows = sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionmajor where workgroupcategory=1 and empname='arvind' and designation='SE' order by empno").collect()
sql("ALTER TABLE partitionmajor COMPACT 'MAJOR'").collect()
- validateDataFiles("default_partitionmajor", "0.2", 1)
+ checkExistence(sql("show segments for table partitionmajor"), true, "0.2")
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionmajor where workgroupcategory=1 and empname='arvind' and designation='SE' order by empno"),
rows)
}
@@ -158,9 +134,7 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
val p1 = sql(s"""select count(*) from staticpartition where deptname='software'""").collect()
val p2 = sql(s"""select count(*) from staticpartition where deptname='finance'""").collect()
sql("ALTER TABLE staticpartition COMPACT 'MINOR'").collect()
-
- validateDataFiles("default_staticpartition", "0.1", 1)
-
+ checkExistence(sql("show segments for table staticpartition"), true, "0.1")
checkAnswer(sql(s"""select count(*) from staticpartition where deptname='software'"""), p1)
checkAnswer(sql(s"""select count(*) from staticpartition where deptname='finance'"""), p2)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
index aac823a..6e36623 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableDropTestCase.scala
@@ -204,6 +204,7 @@ class StandardPartitionTableDropTestCase extends QueryTest with BeforeAndAfterAl
sql(s"""select count (*) from partitionone1 where empno=11"""),
sql(s"""select count (*) from originTable where empno=11"""))
sql(s"""ALTER TABLE partitionone1 DROP PARTITION(empno='11')""")
+ sql(s"CLEAN FILES FOR TABLE partitionone1").show()
assert(Files.notExists(Paths.get(TestQueryExecutor.warehouse + "/partitionone1/" + "empno=11"), LinkOption.NOFOLLOW_LINKS))
sql("drop table if exists partitionone1")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 10da906..c8f7be3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -16,20 +16,24 @@
*/
package org.apache.carbondata.spark.testsuite.standardpartition
+import scala.collection.JavaConverters._
import java.io.{File, FileWriter, IOException}
import java.util
import java.util.concurrent.{Callable, ExecutorService, Executors}
import org.apache.commons.io.FileUtils
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.BatchedDataSourceScanExec
+import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{AnalysisException, CarbonEnv, Row}
+import org.apache.spark.sql.{AnalysisException, CarbonEnv, CarbonSession, Row}
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.spark.rdd.CarbonScanRDD
@@ -66,18 +70,15 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originMultiLoads OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
}
- def validateDataFiles(tableUniqueName: String, segmentId: String, partitions: Int): Unit = {
+ def validateDataFiles(tableUniqueName: String, segmentId: String, partition: Int): Unit = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
- val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
- val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
- override def accept(file: CarbonFile): Boolean = {
- return file.getName.endsWith(".partitionmap")
- }
- })
- assert(dataFiles.length == partitions)
+ val partitions = CarbonFilters
+ .getPartitions(Seq.empty,
+ sqlContext.sparkSession,
+ TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName)))
+ assert(partitions.get.length == partition)
}
test("data loading for partition table for one partition column") {
@@ -92,7 +93,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
- validateDataFiles("default_partitionone", "0", 1)
+ validateDataFiles("default_partitionone", "0", 10)
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionone order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
@@ -111,7 +112,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitiontwo OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
- validateDataFiles("default_partitiontwo", "0", 1)
+ validateDataFiles("default_partitiontwo", "0", 10)
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitiontwo order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
@@ -130,7 +131,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionthree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
- validateDataFiles("default_partitionthree", "0", 1)
+ validateDataFiles("default_partitionthree", "0", 10)
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionthree order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originTable order by empno"))
@@ -146,14 +147,14 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
| utilization int,salary int)
| PARTITIONED BY (workgroupcategory int, empname String, designation String)
| STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('DICTIONARY_INCLUDE'='empname,designation,deptname')
+ | TBLPROPERTIES('DICTIONARY_INCLUDE'='deptname')
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionmultiplethree OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
- validateDataFiles("default_partitionmultiplethree", "1", 1)
- validateDataFiles("default_partitionmultiplethree", "2", 1)
+ validateDataFiles("default_partitionmultiplethree", "1", 10)
+ validateDataFiles("default_partitionmultiplethree", "2", 10)
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from partitionmultiplethree order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originMultiLoads order by empno"))
}
@@ -172,7 +173,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
sql(s"""insert into insertpartitionthree select empno,doj,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
sql(s"""insert into insertpartitionthree select empno,doj,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
- validateDataFiles("default_insertpartitionthree", "0", 1)
+ validateDataFiles("default_insertpartitionthree", "0", 10)
checkAnswer(sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from insertpartitionthree order by empno"),
sql("select empno, empname, designation, doj, workgroupcategory, workgroupcategoryname, deptno, deptname, projectcode, projectjoindate, projectenddate, attendance, utilization, salary from originMultiLoads order by empno"))
@@ -205,7 +206,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE singlepasspartitionone OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"', 'SINGLE_PASS'='true')""")
- validateDataFiles("default_singlepasspartitionone", "0", 1)
+ validateDataFiles("default_singlepasspartitionone", "0", 8)
}
test("data loading for partition table for one static partition column with load syntax") {
@@ -289,7 +290,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
| utilization int,salary int)
| PARTITIONED BY (workgroupcategory int, empname String, designation String)
| STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('DICTIONARY_INCLUDE'='empname,designation,deptname')
+ | TBLPROPERTIES('DICTIONARY_INCLUDE'='deptname')
""".stripMargin)
val tasks = new util.ArrayList[Callable[String]]()
@@ -334,13 +335,12 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_mergeindexpartitionthree")
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", "0")
- val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
- val files = carbonFile.listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = CarbonTablePath.isCarbonIndexFile(file.getName)
- })
- assert(files.length == 10)
+ carbonTable.getTablePath)
+ val details = SegmentStatusManager.readTableStatusFile(tablePath.getTableStatusFilePath)
+ val store = new SegmentFileStore(carbonTable.getTablePath, details(0).getSegmentFile)
+ store.readIndexFiles()
+ store.getIndexFiles
+ assert(store.getIndexFiles.size() == 10)
}
test("load static partition table for one static partition column with load syntax issue") {
@@ -433,10 +433,10 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
}
sql(s"LOAD DATA LOCAL INPATH '$inputPath' INTO TABLE smallpartitionfiles")
FileUtils.deleteDirectory(folder)
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "smallpartitionfiles")
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getSegmentDir("0", "0")
- assert(new File(segmentDir).listFiles().length < 50)
+ val specs = CarbonFilters.getPartitions(Seq.empty, sqlContext.sparkSession, TableIdentifier("smallpartitionfiles"))
+ specs.get.foreach{s =>
+ assert(new File(s.getLocation.toString).listFiles().length < 10)
+ }
}
test("verify partition read with small files") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
index 841185b..c24a277 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
@@ -61,7 +61,6 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
sql(s"""insert into staticpartitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
sql(s"""insert overwrite table staticpartitiondateinsert PARTITION(projectenddate='2016-06-29',doj='2010-12-29 00:00:00') select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary from originTable where projectenddate=cast('2016-06-29' as Date)""")
-// sql(s"""insert overwrite table partitiondateinsert select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable""")
checkAnswer(sql("select * from staticpartitiondateinsert where projectenddate=cast('2016-06-29' as Date)"),
sql("select empno, empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from originTable where projectenddate=cast('2016-06-29' as Date)"))
}
@@ -119,6 +118,7 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
}
test("dynamic and static partition table with overwrite ") {
+ sql("drop table if exists insertstaticpartitiondynamic")
sql(
"""
| CREATE TABLE insertstaticpartitiondynamic (designation String, doj Timestamp,salary int)
@@ -137,6 +137,80 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
}
+ test("dynamic and static partition table with many partition cols overwrite ") {
+ sql("drop table if exists insertstaticpartitiondynamic")
+ sql(
+ """
+ | CREATE TABLE insertstaticpartitiondynamic (designation String,salary int)
+ | PARTITIONED BY (empno int, empname String, doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE insertstaticpartitiondynamic PARTITION(empno, empname, doj) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val rows = sql(s"select count(*) from insertstaticpartitiondynamic").collect()
+ sql("""insert overwrite table insertstaticpartitiondynamic PARTITION(empno='1', empname='ravi', doj) select designation, salary, doj from insertstaticpartitiondynamic""")
+
+ checkAnswer(sql(s"select count(*) from insertstaticpartitiondynamic where empno=1 and empname='ravi'"), rows)
+ }
+
+ test("dynamic and static partition table with many partition cols overwrite with diffrent order") {
+ sql("drop table if exists insertstaticpartitiondynamic")
+ sql(
+ """
+ | CREATE TABLE insertstaticpartitiondynamic (designation String,salary int)
+ | PARTITIONED BY (empno int, empname String, doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE insertstaticpartitiondynamic PARTITION(empno, empname, doj) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val rows = sql(s"select count(*) from insertstaticpartitiondynamic").collect()
+ sql("""insert overwrite table insertstaticpartitiondynamic PARTITION(empno='1', empname, doj) select designation, salary,empname, doj from insertstaticpartitiondynamic""")
+
+ checkAnswer(sql(s"select count(*) from insertstaticpartitiondynamic where empno=1"), rows)
+ }
+
+ test("dynamic and static partition table with many partition cols load overwrite ") {
+ sql("drop table if exists insertstaticpartitiondynamic")
+ sql(
+ """
+ | CREATE TABLE insertstaticpartitiondynamic (designation String,salary int)
+ | PARTITIONED BY (empno1 int, empname1 String, doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE insertstaticpartitiondynamic PARTITION(empno1='1', empname1='ravi', doj) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql(s"select count(*) from insertstaticpartitiondynamic where empno1=1 and empname1='ravi'"), Seq(Row(10)))
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE insertstaticpartitiondynamic PARTITION(empno1='1', empname1='ravi', doj) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql(s"select count(*) from insertstaticpartitiondynamic where empno1=1 and empname1='ravi'"), Seq(Row(10)))
+ checkAnswer(sql(s"select count(*) from insertstaticpartitiondynamic"), Seq(Row(10)))
+
+ intercept[Exception] {
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE insertstaticpartitiondynamic PARTITION(empno1='1', empname1, doj) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ }
+ }
+
+ test("dynamic and static partition table with many partition cols load differnt combinations ") {
+ sql("drop table if exists insertstaticpartitiondynamic")
+ sql(
+ """
+ | CREATE TABLE insertstaticpartitiondynamic (designation String,salary int)
+ | PARTITIONED BY (empno1 int, empname String, doj Timestamp)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE insertstaticpartitiondynamic PARTITION(empno1='1', empname, doj) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql(s"select count(*) from insertstaticpartitiondynamic where empno1=1"), Seq(Row(10)))
+
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' OVERWRITE INTO TABLE insertstaticpartitiondynamic PARTITION(empno1='1', empname, doj) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+
+ checkAnswer(sql(s"select count(*) from insertstaticpartitiondynamic where empno1=1"), Seq(Row(10)))
+ checkAnswer(sql(s"select count(*) from insertstaticpartitiondynamic"), Seq(Row(10)))
+
+ intercept[Exception] {
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE insertstaticpartitiondynamic PARTITION(empno1, empname, doj) OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ }
+ }
+
test("overwriting all partition on table and do compaction") {
sql(
"""
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
index 95345de..918bbff 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
@@ -22,6 +22,7 @@ import org.apache.spark.sql.{DataFrame, Row}
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.rdd.CarbonScanRDD
@@ -260,16 +261,81 @@ test("Creation of partition table should fail if the colname in table schema and
}
}
- test("add partition based on location on partition table should fail"){
+
+ test("add partition based on location on partition table"){
sql("drop table if exists partitionTable")
sql(
"""create table partitionTable (id int,name String) partitioned by(email string) stored by 'carbondata'
""".stripMargin)
sql("insert into partitionTable select 1,'huawei','abc'")
+ val location = metastoredb +"/" +"def"
checkAnswer(sql("show partitions partitionTable"), Seq(Row("email=abc")))
- intercept[Exception]{
- sql("alter table partitionTable add partition (email='def') location 'abc/part1'")
- }
+ sql(s"""alter table partitionTable add partition (email='def') location '$location'""")
+ sql("insert into partitionTable select 1,'huawei','def'")
+ checkAnswer(sql("select email from partitionTable"), Seq(Row("def"), Row("abc")))
+ FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(location))
+ }
+
+ test("add partition with static column partition with load command") {
+ sql(
+ """
+ | CREATE TABLE staticpartitionlocload (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int,projectenddate Date,doj Timestamp)
+ | PARTITIONED BY (empname String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ val location = metastoredb +"/" +"ravi"
+ sql(s"""alter table staticpartitionlocload add partition (empname='ravi') location '$location'""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE staticpartitionlocload partition(empname='ravi') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val frame = sql("select empno,empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from staticpartitionlocload")
+ verifyPartitionInfo(frame, Seq("empname=ravi"))
+ assert(frame.count() == 10)
+ val file = FileFactory.getCarbonFile(location)
+ assert(file.exists())
+ FileFactory.deleteAllCarbonFilesOfDir(file)
+ }
+
+ test("add external partition with static column partition with load command") {
+
+ sql(
+ """
+ | CREATE TABLE staticpartitionlocloadother (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int,projectenddate Date,doj Timestamp)
+ | PARTITIONED BY (empname String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ val location = metastoredb +"/" +"ravi"
+ sql(s"""alter table staticpartitionlocloadother add partition (empname='ravi') location '$location'""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE staticpartitionlocloadother partition(empname='ravi') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE staticpartitionlocloadother partition(empname='indra') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ sql(
+ """
+ | CREATE TABLE staticpartitionextlocload (empno int, designation String,
+ | workgroupcategory int, workgroupcategoryname String, deptno int,
+ | projectjoindate Timestamp,attendance int,
+ | deptname String,projectcode int,
+ | utilization int,salary int,projectenddate Date,doj Timestamp)
+ | PARTITIONED BY (empname String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(s"""alter table staticpartitionextlocload add partition (empname='ravi') location '$location'""")
+ val frame = sql("select empno,empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from staticpartitionextlocload")
+ verifyPartitionInfo(frame, Seq("empname=ravi"))
+ assert(frame.count() == 10)
+ val location2 = storeLocation +"/staticpartitionlocloadother/empname=indra"
+ sql(s"""alter table staticpartitionextlocload add partition (empname='indra') location '$location2'""")
+ val frame1 = sql("select empno,empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from staticpartitionextlocload")
+ verifyPartitionInfo(frame1, Seq("empname=indra"))
+ assert(frame1.count() == 20)
+ val file = FileFactory.getCarbonFile(location)
+ assert(file.exists())
+ FileFactory.deleteAllCarbonFilesOfDir(file)
}
test("drop partition on preAggregate table should fail"){
@@ -295,7 +361,7 @@ test("Creation of partition table should fail if the colname in table schema and
.asInstanceOf[CarbonScanRDD]
}
assert(scanRDD.nonEmpty)
- assert(!partitionNames.map(f => scanRDD.head.partitionNames.exists(_.equals(f))).exists(!_))
+ assert(!partitionNames.map(f => scanRDD.head.partitionNames.exists(_.getPartitions.contains(f))).exists(!_))
}
override def afterAll = {
@@ -318,6 +384,9 @@ test("Creation of partition table should fail if the colname in table schema and
sql("drop table if exists badrecordsPartitionintnull")
sql("drop table if exists badrecordsPartitionintnullalt")
sql("drop table if exists partitionTable")
+ sql("drop table if exists staticpartitionlocload")
+ sql("drop table if exists staticpartitionextlocload")
+ sql("drop table if exists staticpartitionlocloadother")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java
index be459ac..3670e11 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/SparkDataTypeConverterImpl.java
@@ -31,19 +31,31 @@ public final class SparkDataTypeConverterImpl implements DataTypeConverter, Seri
private static final long serialVersionUID = -4379212832935070583L;
public Object convertToDecimal(Object data) {
+ if (null == data) {
+ return null;
+ }
java.math.BigDecimal javaDecVal = new java.math.BigDecimal(data.toString());
return org.apache.spark.sql.types.Decimal.apply(javaDecVal);
}
public byte[] convertFromStringToByte(Object data) {
+ if (null == data) {
+ return null;
+ }
return UTF8String.fromString((String) data).getBytes();
}
public Object convertFromByteToUTF8String(Object data) {
+ if (null == data) {
+ return null;
+ }
return UTF8String.fromBytes((byte[]) data);
}
public Object convertFromStringToUTF8String(Object data) {
+ if (null == data) {
+ return null;
+ }
return UTF8String.fromString((String) data);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index c02ba0a..5fc7e3d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -29,8 +29,9 @@ import org.apache.spark.unsafe.types.UTF8String
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, PartitionMapFileStore}
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, SegmentFileStore}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
@@ -116,7 +117,7 @@ object CarbonStore {
tablePath: String,
carbonTable: CarbonTable,
forceTableClean: Boolean,
- currentTablePartitions: Option[Seq[String]] = None): Unit = {
+ currentTablePartitions: Option[Seq[PartitionSpec]] = None): Unit = {
LOGGER.audit(s"The clean files request has been received for $dbName.$tableName")
var carbonCleanFilesLock: ICarbonLock = null
val absoluteTableIdentifier = if (forceTableClean) {
@@ -139,11 +140,14 @@ object CarbonStore {
CarbonLockUtil
.getLockObject(absoluteTableIdentifier, LockUsage.CLEAN_FILES_LOCK, errorMsg)
DataLoadingUtil.deleteLoadsAndUpdateMetadata(
- isForceDeletion = true, carbonTable)
+ isForceDeletion = true, carbonTable, currentTablePartitions.map(_.asJava).orNull)
CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, true)
currentTablePartitions match {
case Some(partitions) =>
- new PartitionMapFileStore().cleanSegments(carbonTable, partitions.asJava, true)
+ SegmentFileStore.cleanSegments(
+ carbonTable,
+ currentTablePartitions.map(_.asJava).orNull,
+ true)
case _ =>
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
index 9ea58a9..07a2e57 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDropPartitionRDD.scala
@@ -23,10 +23,11 @@ import scala.collection.JavaConverters._
import org.apache.spark.{Partition, SparkContext, TaskContext}
-import org.apache.carbondata.core.metadata.PartitionMapFileStore
-import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.indexstore.PartitionSpec
+import org.apache.carbondata.core.metadata.SegmentFileStore
-case class CarbonDropPartition(rddId: Int, val idx: Int, segmentPath: String)
+case class CarbonDropPartition(rddId: Int, val idx: Int, segment: Segment)
extends Partition {
override val index: Int = idx
@@ -35,115 +36,52 @@ case class CarbonDropPartition(rddId: Int, val idx: Int, segmentPath: String)
}
/**
- * RDD to drop the partitions from partition mapper files of all segments.
+ * RDD to drop the partitions from segment files of all segments.
* @param sc
* @param tablePath
- * @param segments segments to be merged
- * @param partialMatch If it is true then even the partial partition spec matches also can be
- * dropped
+ * @param segments segments to be cleaned
*/
class CarbonDropPartitionRDD(
sc: SparkContext,
tablePath: String,
- segments: Seq[String],
- partitions: Seq[String],
- uniqueId: String,
- partialMatch: Boolean)
- extends CarbonRDD[String](sc, Nil) {
+ segments: Seq[Segment],
+ partitions: util.List[PartitionSpec],
+ uniqueId: String)
+ extends CarbonRDD[(String, String)](sc, Nil) {
override def getPartitions: Array[Partition] = {
segments.zipWithIndex.map {s =>
- CarbonDropPartition(id, s._2, CarbonTablePath.getSegmentPath(tablePath, s._1))
+ CarbonDropPartition(id, s._2, s._1)
}.toArray
}
- override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[String] = {
- val iter = new Iterator[String] {
+ override def internalCompute(
+ theSplit: Partition,
+ context: TaskContext): Iterator[(String, String)] = {
+ val iter = new Iterator[(String, String)] {
val split = theSplit.asInstanceOf[CarbonDropPartition]
- logInfo("Dropping partition information from : " + split.segmentPath)
- partitions.toList.asJava
- val partitionList = new util.ArrayList[util.List[String]]()
- partitionList.add(partitions.toList.asJava)
- new PartitionMapFileStore().dropPartitions(
- split.segmentPath,
- partitionList,
+ logInfo("Dropping partition information from : " + split.segment)
+ val toBeDeletedSegments = new util.ArrayList[String]()
+ val toBeUpdateSegments = new util.ArrayList[String]()
+ new SegmentFileStore(
+ tablePath,
+ split.segment.getSegmentFileName).dropPartitions(
+ split.segment,
+ partitions,
uniqueId,
- partialMatch)
+ toBeDeletedSegments,
+ toBeUpdateSegments)
- var havePair = false
var finished = false
override def hasNext: Boolean = {
- if (!finished && !havePair) {
- finished = true
- havePair = !finished
- }
!finished
}
- override def next(): String = {
- if (!hasNext) {
- throw new java.util.NoSuchElementException("End of stream")
- }
- havePair = false
- ""
+ override def next(): (String, String) = {
+ finished = true
+ (toBeUpdateSegments.asScala.mkString(","), toBeDeletedSegments.asScala.mkString(","))
}
-
- }
- iter
- }
-
-}
-
-/**
- * This RDD is used for committing the partitions which were removed in before step. It just removes
- * old mapper files and related data files.
- * @param sc
- * @param tablePath
- * @param segments segments to be merged
- */
-class CarbonDropPartitionCommitRDD(
- sc: SparkContext,
- tablePath: String,
- segments: Seq[String],
- success: Boolean,
- uniqueId: String,
- partitions: Seq[String])
- extends CarbonRDD[String](sc, Nil) {
-
- override def getPartitions: Array[Partition] = {
- segments.zipWithIndex.map {s =>
- CarbonDropPartition(id, s._2, CarbonTablePath.getSegmentPath(tablePath, s._1))
- }.toArray
- }
-
- override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[String] = {
- val iter = new Iterator[String] {
- val split = theSplit.asInstanceOf[CarbonDropPartition]
- logInfo("Commit partition information from : " + split.segmentPath)
-
- new PartitionMapFileStore().commitPartitions(split.segmentPath, uniqueId, success, tablePath,
- partitions.toList.asJava)
-
- var havePair = false
- var finished = false
-
- override def hasNext: Boolean = {
- if (!finished && !havePair) {
- finished = true
- havePair = !finished
- }
- !finished
- }
-
- override def next(): String = {
- if (!hasNext) {
- throw new java.util.NoSuchElementException("End of stream")
- }
- havePair = false
- ""
- }
-
}
iter
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 0859f2e..e0dcffd 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -26,6 +26,7 @@ import scala.collection.JavaConverters._
import scala.collection.mutable
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.Job
import org.apache.spark._
@@ -37,6 +38,7 @@ import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block._
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
@@ -54,7 +56,7 @@ import org.apache.carbondata.processing.merger._
import org.apache.carbondata.processing.splits.TableSplit
import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil, CarbonLoaderUtil}
import org.apache.carbondata.spark.MergeResult
-import org.apache.carbondata.spark.util.{CommonUtil, SparkDataTypeConverterImpl, Util}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, SparkDataTypeConverterImpl, Util}
class CarbonMergerRDD[K, V](
sc: SparkContext,
@@ -87,8 +89,8 @@ class CarbonMergerRDD[K, V](
} else {
carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
}
- val partitionNames = if (carbonTable.isHivePartitionTable) {
- carbonSparkPartition.partitionNames.get.asJava
+ val partitionSpec = if (carbonTable.isHivePartitionTable) {
+ carbonSparkPartition.partitionSpec.get
} else {
null
}
@@ -138,6 +140,14 @@ class CarbonMergerRDD[K, V](
)
}
carbonLoadModel.setSegmentId(mergeNumber)
+
+ if(carbonTable.isHivePartitionTable) {
+ carbonLoadModel.setTaskNo(
+ CarbonScalaUtil.generateUniqueNumber(
+ theSplit.index,
+ mergeNumber.replace(".", ""), 0L))
+ }
+
CommonUtil.setTempStoreLocation(theSplit.index, carbonLoadModel, true, false)
// get destination segment properties as sent from driver which is of last segment.
@@ -198,7 +208,7 @@ class CarbonMergerRDD[K, V](
segmentProperties,
carbonMergerMapping.campactionType,
factTableName,
- partitionNames)
+ partitionSpec)
} else {
LOGGER.info("RowResultMergerProcessor flow is selected")
processor =
@@ -209,7 +219,7 @@ class CarbonMergerRDD[K, V](
tempStoreLoc,
carbonLoadModel,
carbonMergerMapping.campactionType,
- partitionNames)
+ partitionSpec)
}
mergeStatus = processor.execute(result2)
mergeResult = tableBlockInfoList.get(0).getSegmentId + ',' + mergeNumber
@@ -260,7 +270,7 @@ class CarbonMergerRDD[K, V](
val format = CarbonInputFormatUtil.createCarbonInputFormat(absoluteTableIdentifier, job)
CarbonTableInputFormat.setPartitionsToPrune(
job.getConfiguration,
- carbonMergerMapping.currentPartitions.asJava)
+ carbonMergerMapping.currentPartitions.map(_.asJava).orNull)
CarbonTableInputFormat.setTableInfo(job.getConfiguration,
carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getTableInfo)
var updateDetails: UpdateVO = null
@@ -284,10 +294,10 @@ class CarbonMergerRDD[K, V](
for (eachSeg <- carbonMergerMapping.validSegments) {
// map for keeping the relation of a task and its blocks.
- job.getConfiguration.set(CarbonTableInputFormat.INPUT_SEGMENT_NUMBERS, eachSeg)
+ job.getConfiguration.set(CarbonTableInputFormat.INPUT_SEGMENT_NUMBERS, eachSeg.getSegmentNo)
if (updateStatusManager.getUpdateStatusDetails.length != 0) {
- updateDetails = updateStatusManager.getInvalidTimestampRange(eachSeg)
+ updateDetails = updateStatusManager.getInvalidTimestampRange(eachSeg.getSegmentNo)
}
val updated: Boolean = updateStatusManager.getUpdateStatusDetails.length != 0
@@ -304,7 +314,7 @@ class CarbonMergerRDD[K, V](
val blockInfo = new TableBlockInfo(entry.getPath.toString,
entry.getStart, entry.getSegmentId,
entry.getLocations, entry.getLength, entry.getVersion,
- updateStatusManager.getDeleteDeltaFilePath(entry.getPath.toString)
+ updateStatusManager.getDeleteDeltaFilePath(entry.getPath.toString, entry.getSegmentId)
)
(!updated || (updated && (!CarbonUtil
.isInvalidTableBlock(blockInfo.getSegmentId, blockInfo.getFilePath,
@@ -329,7 +339,7 @@ class CarbonMergerRDD[K, V](
}
val columnToCardinalityMap = new util.HashMap[java.lang.String, Integer]()
- val partitionTaskMap = new util.HashMap[util.List[String], String]()
+ val partitionTaskMap = new util.HashMap[PartitionSpec, String]()
val counter = new AtomicInteger()
carbonInputSplits.foreach { split =>
val taskNo = getTaskNo(split, partitionTaskMap, counter)
@@ -455,16 +465,20 @@ class CarbonMergerRDD[K, V](
private def getTaskNo(
split: CarbonInputSplit,
- partitionTaskMap: util.Map[List[String], String],
+ partitionTaskMap: util.Map[PartitionSpec, String],
counter: AtomicInteger): String = {
if (carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.isHivePartitionTable) {
- val partitions =
- carbonMergerMapping.partitionMapper.getPartitionMap.get(
- CarbonTablePath.getCarbonIndexFileName(split.getBlockPath))
- var task = partitionTaskMap.get(partitions)
+ val path = split.getPath.getParent
+ val partTask =
+ carbonMergerMapping.currentPartitions.get.find(p => p.getLocation.equals(path)) match {
+ case Some(part) => part
+ case None =>
+ throw new UnsupportedOperationException("Cannot do compaction on dropped partition")
+ }
+ var task = partitionTaskMap.get(partTask)
if (task == null) {
task = counter.incrementAndGet().toString
- partitionTaskMap.put(partitions, task)
+ partitionTaskMap.put(partTask, task)
}
task
} else {
@@ -472,10 +486,12 @@ class CarbonMergerRDD[K, V](
}
}
+
+
private def getPartitionNamesFromTask(taskId: String,
- partitionTaskMap: util.Map[List[String], String]): Option[Seq[String]] = {
+ partitionTaskMap: util.Map[PartitionSpec, String]): Option[PartitionSpec] = {
if (carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.isHivePartitionTable) {
- Some(partitionTaskMap.asScala.find(f => f._2.equals(taskId)).get._1.asScala)
+ Some(partitionTaskMap.asScala.find(f => f._2.equals(taskId)).get._1)
} else {
None
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 8c29c2a..772f702 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -40,6 +40,7 @@ import org.apache.spark.sql.util.SparkSQLUtil.sessionState
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.Distributable
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.TableInfo
import org.apache.carbondata.core.scan.expression.Expression
@@ -68,7 +69,7 @@ class CarbonScanRDD(
@transient serializedTableInfo: Array[Byte],
@transient tableInfo: TableInfo,
inputMetricsStats: InitInputMetrics,
- @transient val partitionNames: Seq[String])
+ @transient val partitionNames: Seq[PartitionSpec])
extends CarbonRDDWithTableInfo[InternalRow](spark.sparkContext, Nil, serializedTableInfo) {
private val queryId = sparkContext.getConf.get("queryId", System.nanoTime() + "")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala
index 036f1d1..b473d35 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonSparkPartition.scala
@@ -19,6 +19,7 @@ package org.apache.carbondata.spark.rdd
import org.apache.spark.{Partition, SerializableWritable}
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.hadoop.CarbonMultiBlockSplit
class CarbonSparkPartition(
@@ -26,7 +27,7 @@ class CarbonSparkPartition(
val idx: Int,
@transient val multiBlockSplit: CarbonMultiBlockSplit,
val partitionId: Int = 0,
- val partitionNames: Option[Seq[String]] = None)
+ val partitionSpec: Option[PartitionSpec] = None)
extends Partition {
val split = new SerializableWritable[CarbonMultiBlockSplit](multiBlockSplit)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
index 2aa5610..c73065d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
@@ -32,7 +32,7 @@ object PartitionDropper {
def triggerPartitionDrop(dropPartitionCallableModel: DropPartitionCallableModel): Unit = {
val alterPartitionModel = new AlterPartitionModel(dropPartitionCallableModel.carbonLoadModel,
- dropPartitionCallableModel.segmentId,
+ dropPartitionCallableModel.segmentId.getSegmentNo,
dropPartitionCallableModel.oldPartitionIds,
dropPartitionCallableModel.sqlContext
)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 73be3c8..33263d6 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -17,27 +17,35 @@
package org.apache.carbondata.spark.util
+import java.{lang, util}
import java.nio.charset.Charset
import java.text.SimpleDateFormat
-import java.util
+import java.util.Date
+import com.univocity.parsers.common.TextParsingException
+import org.apache.spark.SparkException
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.catalog.CatalogTablePartition
import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.execution.command.DataTypeInfo
+import org.apache.spark.sql.execution.command.{DataTypeInfo, UpdateTableModel}
import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
-import org.apache.carbondata.common.constants.LoggerAction
+import org.apache.carbondata.common.logging.LogService
import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
-import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory
+import org.apache.carbondata.core.metadata.ColumnIdentifier
import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, StructField => CarbonStructField}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, ColumnSchema}
-import org.apache.carbondata.core.util.{CarbonSessionInfo, DataTypeUtil}
+import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.processing.exception.DataLoadingException
+import org.apache.carbondata.processing.loading.FailureCauses
+import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
object CarbonScalaUtil {
def convertSparkToCarbonDataType(dataType: DataType): CarbonDataType = {
@@ -167,33 +175,48 @@ object CarbonScalaUtil {
* @param dataType Datatype to convert and then convert to String
* @param timeStampFormat Timestamp format to convert in case of timestamp datatypes
* @param dateFormat DataFormat to convert in case of DateType datatype
- * @param serializationNullFormat if this encounters in input data then data will
- * be treated as null
* @return converted String
*/
- def convertToString(
+ def convertToDateAndTimeFormats(
value: String,
dataType: DataType,
timeStampFormat: SimpleDateFormat,
- dateFormat: SimpleDateFormat,
- serializationNullFormat: String): String = {
- if (value == null || serializationNullFormat.equals(value)) {
- return null
- }
- dataType match {
- case TimestampType if timeStampFormat != null =>
- DateTimeUtils.timestampToString(timeStampFormat.parse(value).getTime * 1000)
- case DateType if dateFormat != null =>
- DateTimeUtils.dateToString(
- (dateFormat.parse(value).getTime / DateTimeUtils.MILLIS_PER_DAY).toInt)
- case ShortType => value.toShort.toString
- case IntegerType => value.toInt.toString
- case LongType => value.toLong.toString
- case DoubleType => value.toDouble.toString
- case FloatType => value.toFloat.toString
- case d: DecimalType => new java.math.BigDecimal(value).toPlainString
- case BooleanType => value.toBoolean.toString
- case _ => value
+ dateFormat: SimpleDateFormat): String = {
+ val defaultValue = value != null && value.equalsIgnoreCase(hivedefaultpartition)
+ try {
+ dataType match {
+ case TimestampType if timeStampFormat != null =>
+ if (defaultValue) {
+ timeStampFormat.format(new Date())
+ } else {
+ timeStampFormat.format(DateTimeUtils.stringToTime(value))
+ }
+ case DateType if dateFormat != null =>
+ if (defaultValue) {
+ dateFormat.format(new Date())
+ } else {
+ dateFormat.format(DateTimeUtils.stringToTime(value))
+ }
+ case _ =>
+ val convertedValue =
+ DataTypeUtil
+ .getDataBasedOnDataType(value, convertSparkToCarbonDataType(dataType))
+ if (convertedValue == null) {
+ if (defaultValue) {
+ return dataType match {
+ case BooleanType => "false"
+ case _ => "0"
+ }
+ }
+ throw new MalformedCarbonCommandException(
+ s"Value $value with datatype $dataType on static partition is not correct")
+ }
+ value
+ }
+ } catch {
+ case e: Exception =>
+ throw new MalformedCarbonCommandException(
+ s"Value $value with datatype $dataType on static partition is not correct")
}
}
@@ -214,14 +237,20 @@ object CarbonScalaUtil {
val time = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(
column.getDataType,
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
- ).getValueFromSurrogate(value.toInt).toString
- return DateTimeUtils.timestampToString(time.toLong * 1000)
+ ).getValueFromSurrogate(value.toInt)
+ if (time == null) {
+ return null
+ }
+ return DateTimeUtils.timestampToString(time.toString.toLong * 1000)
} else if (column.getDataType.equals(CarbonDataTypes.DATE)) {
val date = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(
column.getDataType,
CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT
- ).getValueFromSurrogate(value.toInt).toString
- return DateTimeUtils.dateToString(date.toInt)
+ ).getValueFromSurrogate(value.toInt)
+ if (date == null) {
+ return null
+ }
+ return DateTimeUtils.dateToString(date.toString.toInt)
}
}
val dictionaryPath =
@@ -271,14 +300,28 @@ object CarbonScalaUtil {
CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT
).generateDirectSurrogateKey(value).toString
}
+ } else if (column.hasEncoding(Encoding.DICTIONARY)) {
+ val cacheProvider: CacheProvider = CacheProvider.getInstance
+ val reverseCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] =
+ cacheProvider.createCache(CacheType.REVERSE_DICTIONARY)
+ val dictionaryPath =
+ table.getTableInfo.getFactTable.getTableProperties.get(
+ CarbonCommonConstants.DICTIONARY_PATH)
+ val dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(
+ table.getAbsoluteTableIdentifier,
+ new ColumnIdentifier(
+ column.getColumnUniqueId,
+ column.getColumnProperties,
+ column.getDataType),
+ column.getDataType,
+ dictionaryPath)
+ return reverseCache.get(dictionaryColumnUniqueIdentifier).getSurrogateKey(value).toString
}
column.getDataType match {
case CarbonDataTypes.TIMESTAMP =>
- DataTypeUtil.getDataDataTypeForNoDictionaryColumn(value,
- column.getDataType,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT).toString
+ DateTimeUtils.stringToTime(value).getTime.toString
case CarbonDataTypes.DATE =>
- DateTimeUtils.stringToDate(UTF8String.fromString(value)).get.toString
+ DateTimeUtils.stringToTime(value).getTime.toString
case _ => value
}
} catch {
@@ -287,7 +330,7 @@ object CarbonScalaUtil {
}
}
- private val hiveignorepartition = "__HIVE_IGNORE_PARTITION__"
+ private val hivedefaultpartition = "__HIVE_DEFAULT_PARTITION__"
/**
* Update partition values as per the right date and time format
@@ -295,10 +338,7 @@ object CarbonScalaUtil {
*/
def updatePartitions(
partitionSpec: Map[String, String],
- table: CarbonTable,
- timeFormat: SimpleDateFormat,
- dateFormat: SimpleDateFormat): Map[String, String] = {
- val hivedefaultpartition = "__HIVE_DEFAULT_PARTITION__"
+ table: CarbonTable): Map[String, String] = {
val cacheProvider: CacheProvider = CacheProvider.getInstance
val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] =
cacheProvider.createCache(CacheType.FORWARD_DICTIONARY)
@@ -340,37 +380,14 @@ object CarbonScalaUtil {
* Update partition values as per the right date and time format
*/
def updatePartitions(
- carbonSessionInfo: CarbonSessionInfo,
parts: Seq[CatalogTablePartition],
table: CarbonTable): Seq[CatalogTablePartition] = {
- val dateFormatStr = carbonSessionInfo.getThreadParams.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
- val dateFormat = new SimpleDateFormat(dateFormatStr)
- val timeFormatStr = carbonSessionInfo.getThreadParams.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT)
- val timeFormat = new SimpleDateFormat(timeFormatStr)
- val serializeFormat = carbonSessionInfo.getThreadParams.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_SERIALIZATION_NULL_FORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_SERIALIZATION_NULL_FORMAT_DEFAULT)
- val isEmptyBadRecord = carbonSessionInfo.getThreadParams.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT).toBoolean
- val badRecordAction = carbonSessionInfo.getThreadParams.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
- LoggerAction.FAIL.toString)
parts.map{ f =>
val changedSpec =
updatePartitions(
f.spec,
- table,
- timeFormat,
- dateFormat)
+ table)
f.copy(spec = changedSpec)
- }.filterNot{ p =>
- // Filter the special bad record ignore case string
- p.spec.exists(_._2.equals(hiveignorepartition))
}.groupBy(p => p.spec).map(f => f._2.head).toSeq // Avoid duplicates by do groupby
}
@@ -508,4 +525,64 @@ object CarbonScalaUtil {
}
}
+ /**
+ * Retrieve error message from exception
+ */
+ def retrieveAndLogErrorMsg(ex: Throwable, logger: LogService): (String, String) = {
+ var errorMessage = "DataLoad failure"
+ var executorMessage = ""
+ if (ex != null) {
+ ex match {
+ case sparkException: SparkException =>
+ if (sparkException.getCause.isInstanceOf[DataLoadingException] ||
+ sparkException.getCause.isInstanceOf[CarbonDataLoadingException]) {
+ executorMessage = sparkException.getCause.getMessage
+ errorMessage = errorMessage + ": " + executorMessage
+ } else if (sparkException.getCause.isInstanceOf[TextParsingException]) {
+ executorMessage = CarbonDataProcessorUtil
+ .trimErrorMessage(sparkException.getCause.getMessage)
+ errorMessage = errorMessage + " : " + executorMessage
+ } else if (sparkException.getCause.isInstanceOf[SparkException]) {
+ val (executorMsgLocal, errorMsgLocal) =
+ retrieveAndLogErrorMsg(sparkException.getCause, logger)
+ executorMessage = executorMsgLocal
+ errorMessage = errorMsgLocal
+ }
+ case aex: AnalysisException =>
+ logger.error(aex.getMessage())
+ throw aex
+ case _ =>
+ if (ex.getCause != null) {
+ executorMessage = ex.getCause.getMessage
+ errorMessage = errorMessage + ": " + executorMessage
+ }
+ }
+ }
+ (executorMessage, errorMessage)
+ }
+
+ /**
+ * Update error inside update model
+ */
+ def updateErrorInUpdateModel(updateModel: UpdateTableModel, executorMessage: String): Unit = {
+ if (updateModel.executorErrors.failureCauses == FailureCauses.NONE) {
+ updateModel.executorErrors.failureCauses = FailureCauses.EXECUTOR_FAILURE
+ if (null != executorMessage && !executorMessage.isEmpty) {
+ updateModel.executorErrors.errorMsg = executorMessage
+ } else {
+ updateModel.executorErrors.errorMsg = "Update failed as the data load has failed."
+ }
+ }
+ }
+
+ /**
+ * Generate unique number to be used as partition number of file name
+ */
+ def generateUniqueNumber(taskId: Int,
+ segmentId: String,
+ partitionNumber: lang.Long): String = {
+ String.valueOf(Math.pow(10, 2).toInt + segmentId.toInt) +
+ String.valueOf(Math.pow(10, 5).toInt + taskId) +
+ String.valueOf(partitionNumber + Math.pow(10, 5).toInt)
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 64e4bb1..94668bd 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -683,7 +683,8 @@ object CommonUtil {
def getCsvHeaderColumns(
carbonLoadModel: CarbonLoadModel,
- hadoopConf: Configuration): Array[String] = {
+ hadoopConf: Configuration,
+ staticPartitionCols: util.List[String] = new util.ArrayList[String]()): Array[String] = {
val delimiter = if (StringUtils.isEmpty(carbonLoadModel.getCsvDelimiter)) {
CarbonCommonConstants.COMMA
} else {
@@ -691,7 +692,7 @@ object CommonUtil {
}
var csvFile: String = null
var csvHeader: String = carbonLoadModel.getCsvHeader
- val csvColumns = if (StringUtils.isBlank(csvHeader)) {
+ var csvColumns = if (StringUtils.isBlank(csvHeader)) {
// read header from csv file
csvFile = carbonLoadModel.getFactFilePath.split(",")(0)
csvHeader = CarbonUtil.readHeader(csvFile, hadoopConf)
@@ -704,7 +705,7 @@ object CommonUtil {
}
if (!CarbonDataProcessorUtil.isHeaderValid(carbonLoadModel.getTableName, csvColumns,
- carbonLoadModel.getCarbonDataLoadSchema)) {
+ carbonLoadModel.getCarbonDataLoadSchema, staticPartitionCols)) {
if (csvFile == null) {
LOGGER.error("CSV header in DDL is not proper."
+ " Column names in schema and CSV header are not the same.")
@@ -720,7 +721,23 @@ object CommonUtil {
+ "the same. Input file : " + CarbonUtil.removeAKSK(csvFile))
}
}
- csvColumns
+ // In case of static partition columns just change the name of header if already exists as
+ // we should not take the column from csv file and add them as new columns at the end.
+ if (staticPartitionCols.size() > 0) {
+ val scalaIgnoreColumns = staticPartitionCols.asScala
+ var updatedCols = csvColumns.map{col =>
+ if (scalaIgnoreColumns.exists(_.equalsIgnoreCase(col))) {
+ col + "1"
+ } else {
+ col
+ }
+ }.toList.asJava
+ updatedCols = new util.ArrayList[String](updatedCols)
+ updatedCols.addAll(staticPartitionCols)
+ updatedCols.asScala.toArray
+ } else {
+ csvColumns
+ }
}
def validateMaxColumns(csvHeaders: Array[String], maxColumns: String): Int = {
@@ -866,7 +883,7 @@ object CommonUtil {
val carbonTable = CarbonMetadata.getInstance
.getCarbonTable(identifier.getCarbonTableIdentifier.getTableUniqueName)
DataLoadingUtil.deleteLoadsAndUpdateMetadata(
- isForceDeletion = true, carbonTable)
+ isForceDeletion = true, carbonTable, null)
} catch {
case _: Exception =>
LOGGER.warn(s"Error while cleaning table " +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index a38eaba..6767ef7 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -18,7 +18,8 @@
package org.apache.carbondata.spark.util
import java.text.SimpleDateFormat
-import java.util.{Date, Locale}
+import java.util
+import java.util.{Date, List, Locale}
import scala.collection.{immutable, mutable}
import scala.collection.JavaConverters._
@@ -43,9 +44,11 @@ import org.apache.spark.sql.util.SparkSQLUtil.sessionState
import org.apache.carbondata.common.constants.LoggerAction
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
+import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants
import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
@@ -221,7 +224,9 @@ object DataLoadingUtil {
options: immutable.Map[String, String],
optionsFinal: mutable.Map[String, String],
carbonLoadModel: CarbonLoadModel,
- hadoopConf: Configuration): Unit = {
+ hadoopConf: Configuration,
+ partition: Map[String, Option[String]] = Map.empty,
+ isDataFrame: Boolean = false): Unit = {
carbonLoadModel.setTableName(table.getTableName)
carbonLoadModel.setDatabaseName(table.getDatabaseName)
carbonLoadModel.setTablePath(table.getTablePath)
@@ -331,8 +336,13 @@ object DataLoadingUtil {
carbonLoadModel.setCsvDelimiter(CarbonUtil.unescapeChar(delimeter))
carbonLoadModel.setCsvHeader(fileHeader)
carbonLoadModel.setColDictFilePath(column_dict)
+
+ val ignoreColumns = new util.ArrayList[String]()
+ if (!isDataFrame) {
+ ignoreColumns.addAll(partition.filter(_._2.isDefined).keys.toList.asJava)
+ }
carbonLoadModel.setCsvHeaderColumns(
- CommonUtil.getCsvHeaderColumns(carbonLoadModel, hadoopConf))
+ CommonUtil.getCsvHeaderColumns(carbonLoadModel, hadoopConf, ignoreColumns))
val validatedMaxColumns = CommonUtil.validateMaxColumns(
carbonLoadModel.getCsvHeaderColumns,
@@ -360,33 +370,40 @@ object DataLoadingUtil {
def deleteLoadsAndUpdateMetadata(
isForceDeletion: Boolean,
- carbonTable: CarbonTable): Unit = {
+ carbonTable: CarbonTable,
+ specs: util.List[PartitionSpec]): Unit = {
if (isLoadDeletionRequired(carbonTable.getMetaDataFilepath)) {
- val details = SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath)
val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
- val carbonTableStatusLock =
- CarbonLockFactory.getCarbonLockObj(
- absoluteTableIdentifier,
- LockUsage.TABLE_STATUS_LOCK
- )
-
- // Delete marked loads
- val isUpdationRequired =
- DeleteLoadFolders.deleteLoadFoldersFromFileSystem(
- absoluteTableIdentifier,
- isForceDeletion,
- details,
- carbonTable.getMetaDataFilepath
- )
- var updationCompletionStaus = false
-
- if (isUpdationRequired) {
+ val (details, updationRequired) =
+ isUpdationRequired(
+ isForceDeletion,
+ carbonTable,
+ absoluteTableIdentifier)
+
+
+ if (updationRequired) {
+ val carbonTableStatusLock =
+ CarbonLockFactory.getCarbonLockObj(
+ absoluteTableIdentifier,
+ LockUsage.TABLE_STATUS_LOCK
+ )
+ var locked = false
+ var updationCompletionStaus = false
try {
// Update load metadate file after cleaning deleted nodes
- if (carbonTableStatusLock.lockWithRetries()) {
+ locked = carbonTableStatusLock.lockWithRetries()
+ if (locked) {
LOGGER.info("Table status lock has been successfully acquired.")
-
+ // Again read status and check to verify updation required or not.
+ val (details, updationRequired) =
+ isUpdationRequired(
+ isForceDeletion,
+ carbonTable,
+ absoluteTableIdentifier)
+ if (!updationRequired) {
+ return
+ }
// read latest table status again.
val latestMetadata = SegmentStatusManager
.readLoadMetadata(carbonTable.getMetaDataFilepath)
@@ -409,17 +426,34 @@ object DataLoadingUtil {
}
updationCompletionStaus = true
} finally {
- CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK)
+ if (locked) {
+ CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK)
+ }
}
if (updationCompletionStaus) {
DeleteLoadFolders
.physicalFactAndMeasureMetadataDeletion(absoluteTableIdentifier,
- carbonTable.getMetaDataFilepath, isForceDeletion)
+ carbonTable.getMetaDataFilepath, isForceDeletion, specs)
}
}
}
}
+ private def isUpdationRequired(isForceDeletion: Boolean,
+ carbonTable: CarbonTable,
+ absoluteTableIdentifier: AbsoluteTableIdentifier) = {
+ val details = SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath)
+ // Delete marked loads
+ val isUpdationRequired =
+ DeleteLoadFolders.deleteLoadFoldersFromFileSystem(
+ absoluteTableIdentifier,
+ isForceDeletion,
+ details,
+ carbonTable.getMetaDataFilepath
+ )
+ (details, isUpdationRequired)
+ }
+
/**
* creates a RDD that does reading of multiple CSV files
*/
[03/16] carbondata git commit: [CARBONDATA-2185] Add InputMetrics for
Streaming Reader
Posted by gv...@apache.org.
[CARBONDATA-2185] Add InputMetrics for Streaming Reader
This closes #1985
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4bbbd4b1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4bbbd4b1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4bbbd4b1
Branch: refs/heads/branch-1.3
Commit: 4bbbd4b1df444163cfb72cf74a05c1a9d09e1200
Parents: f8a62a9
Author: BJangir <ba...@gmail.com>
Authored: Mon Feb 19 22:31:00 2018 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Tue Feb 27 12:46:29 2018 +0530
----------------------------------------------------------------------
.../streaming/CarbonStreamRecordReader.java | 18 ++++++++++++++++++
.../carbondata/spark/rdd/CarbonScanRDD.scala | 1 +
2 files changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bbbd4b1/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
index 773089b..95a7af0 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
@@ -58,6 +58,7 @@ import org.apache.carbondata.format.BlockletHeader;
import org.apache.carbondata.format.FileHeader;
import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
+import org.apache.carbondata.hadoop.InputMetricsStats;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
import org.apache.carbondata.hadoop.util.CarbonTypeUtil;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -137,6 +138,9 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
// return raw row for handoff
private boolean useRawRow = false;
+ // InputMetricsStats
+ private InputMetricsStats inputMetricsStats;
+
@Override public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
// input
@@ -392,8 +396,18 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
@Override public Object getCurrentValue() throws IOException, InterruptedException {
if (isVectorReader) {
+ int value = columnarBatch.numValidRows();
+ if (inputMetricsStats != null) {
+ inputMetricsStats.incrementRecordRead((long) value);
+ }
+
return columnarBatch;
}
+
+ if (inputMetricsStats != null) {
+ inputMetricsStats.incrementRecordRead(1L);
+ }
+
return outputRow;
}
@@ -730,6 +744,10 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
this.isVectorReader = isVectorReader;
}
+ public void setInputMetricsStats(InputMetricsStats inputMetricsStats) {
+ this.inputMetricsStats = inputMetricsStats;
+ }
+
@Override public void close() throws IOException {
if (null != input) {
input.close();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bbbd4b1/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index e554a58..8c29c2a 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -345,6 +345,7 @@ class CarbonScanRDD(
val streamReader = inputFormat.createRecordReader(inputSplit, attemptContext)
.asInstanceOf[CarbonStreamRecordReader]
streamReader.setVectorReader(vectorReader)
+ streamReader.setInputMetricsStats(inputMetricsStats)
model.setStatisticsRecorder(
CarbonTimeStatisticsFactory.createExecutorRecorder(model.getQueryId))
streamReader.setQueryModel(model)
[12/16] carbondata git commit: [CARBONDATA-2187][PARTITION] Partition
restructure for new folder structure and supporting partition location
feature
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index bc84e04..20d3032 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -29,8 +29,10 @@ import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.CarbonTableIdentifier
-import org.apache.carbondata.core.metadata.PartitionMapFileStore.PartitionMapper
+import org.apache.carbondata.core.metadata.SegmentFileStore.SegmentFile
import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes, DecimalType}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema._
@@ -109,15 +111,14 @@ case class CarbonMergerMapping(
var mergedLoadName: String,
databaseName: String,
factTableName: String,
- validSegments: Array[String],
+ validSegments: Array[Segment],
tableId: String,
campactionType: CompactionType,
// maxSegmentColCardinality is Cardinality of last segment of compaction
var maxSegmentColCardinality: Array[Int],
// maxSegmentColumnSchemaList is list of column schema of last segment of compaction
var maxSegmentColumnSchemaList: List[ColumnSchema],
- currentPartitions: Seq[String],
- @transient partitionMapper: PartitionMapper)
+ currentPartitions: Option[Seq[PartitionSpec]])
case class NodeInfo(TaskId: String, noOfBlocks: Int)
@@ -133,20 +134,20 @@ case class UpdateTableModel(
isUpdate: Boolean,
updatedTimeStamp: Long,
var executorErrors: ExecutionErrors,
- deletedSegments: Seq[String])
+ deletedSegments: Seq[Segment])
case class CompactionModel(compactionSize: Long,
compactionType: CompactionType,
carbonTable: CarbonTable,
isDDLTrigger: Boolean,
- currentPartitions: Seq[String])
+ currentPartitions: Option[Seq[PartitionSpec]])
case class CompactionCallableModel(carbonLoadModel: CarbonLoadModel,
carbonTable: CarbonTable,
loadsToMerge: util.List[LoadMetadataDetails],
sqlContext: SQLContext,
compactionType: CompactionType,
- currentPartitions: Seq[String])
+ currentPartitions: Option[Seq[PartitionSpec]])
case class AlterPartitionModel(carbonLoadModel: CarbonLoadModel,
segmentId: String,
@@ -161,7 +162,7 @@ case class SplitPartitionCallableModel(carbonLoadModel: CarbonLoadModel,
sqlContext: SQLContext)
case class DropPartitionCallableModel(carbonLoadModel: CarbonLoadModel,
- segmentId: String,
+ segmentId: Segment,
partitionId: String,
oldPartitionIds: List[Int],
dropWithData: Boolean,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 8ed7623..1695a13 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -38,8 +38,8 @@ import org.apache.spark.{SparkEnv, SparkException, TaskContext}
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, NewHadoopRDD, RDD}
import org.apache.spark.sql.{AnalysisException, CarbonEnv, DataFrame, Row, SQLContext}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.{CompactionModel, ExecutionErrors, UpdateTableModel}
-import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
import org.apache.spark.sql.hive.DistributionUtil
import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.util.CarbonException
@@ -47,6 +47,7 @@ import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.common.constants.LoggerAction
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
@@ -207,7 +208,9 @@ object CarbonDataRDDFactory {
compactionType,
table,
compactionModel.isDDLTrigger,
- CarbonFilters.getCurrentPartitions(sqlContext.sparkSession, table))
+ CarbonFilters.getCurrentPartitions(sqlContext.sparkSession,
+ TableIdentifier(table.getTableName,
+ Some(table.getDatabaseName))))
// proceed for compaction
try {
CompactionFactory.getCompactor(
@@ -395,26 +398,9 @@ object CarbonDataRDDFactory {
} catch {
case ex: Throwable =>
loadStatus = SegmentStatus.LOAD_FAILURE
- ex match {
- case sparkException: SparkException =>
- if (sparkException.getCause.isInstanceOf[DataLoadingException] ||
- sparkException.getCause.isInstanceOf[CarbonDataLoadingException]) {
- executorMessage = sparkException.getCause.getMessage
- errorMessage = errorMessage + ": " + executorMessage
- } else if (sparkException.getCause.isInstanceOf[TextParsingException]) {
- executorMessage = CarbonDataProcessorUtil
- .trimErrorMessage(sparkException.getCause.getMessage)
- errorMessage = errorMessage + " : " + executorMessage
- }
- case aex: AnalysisException =>
- LOGGER.error(aex.getMessage())
- throw aex
- case _ =>
- if (ex.getCause != null) {
- executorMessage = ex.getCause.getMessage
- errorMessage = errorMessage + ": " + executorMessage
- }
- }
+ val (extrMsgLocal, errorMsgLocal) = CarbonScalaUtil.retrieveAndLogErrorMsg(ex, LOGGER)
+ executorMessage = extrMsgLocal
+ errorMessage = errorMsgLocal
LOGGER.info(errorMessage)
LOGGER.error(ex)
} finally {
@@ -423,14 +409,7 @@ object CarbonDataRDDFactory {
// handle the status file updation for the update cmd.
if (updateModel.isDefined) {
if (loadStatus == SegmentStatus.LOAD_FAILURE) {
- if (updateModel.get.executorErrors.failureCauses == FailureCauses.NONE) {
- updateModel.get.executorErrors.failureCauses = FailureCauses.EXECUTOR_FAILURE
- if (null != executorMessage && !executorMessage.isEmpty) {
- updateModel.get.executorErrors.errorMsg = executorMessage
- } else {
- updateModel.get.executorErrors.errorMsg = "Update failed as the data load has failed."
- }
- }
+ CarbonScalaUtil.updateErrorInUpdateModel(updateModel.get, executorMessage)
return
} else if (loadStatus == SegmentStatus.LOAD_PARTIAL_SUCCESS &&
updateModel.get.executorErrors.failureCauses == FailureCauses.BAD_RECORDS &&
@@ -441,12 +420,12 @@ object CarbonDataRDDFactory {
// success case.
// write the dictionary file in case of single_pass true
writeDictionary(carbonLoadModel, result, false)
- val segmentDetails = new util.HashSet[String]()
+ val segmentDetails = new util.HashSet[Segment]()
var resultSize = 0
res.foreach { resultOfSeg =>
resultSize = resultSize + resultOfSeg.size
resultOfSeg.foreach { resultOfBlock =>
- segmentDetails.add(resultOfBlock._2._1.getLoadName)
+ segmentDetails.add(new Segment(resultOfBlock._2._1.getLoadName, null))
}
}
@@ -462,7 +441,7 @@ object CarbonDataRDDFactory {
carbonTable,
updateModel.get.updatedTimeStamp + "",
true,
- new util.ArrayList[String](0))) {
+ new util.ArrayList[Segment](0))) {
LOGGER.audit("Data update is successful for " +
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
} else {
@@ -744,7 +723,9 @@ object CarbonDataRDDFactory {
CompactionType.MINOR,
carbonTable,
isCompactionTriggerByDDl,
- CarbonFilters.getCurrentPartitions(sqlContext.sparkSession, carbonTable))
+ CarbonFilters.getCurrentPartitions(sqlContext.sparkSession,
+ TableIdentifier(carbonTable.getTableName,
+ Some(carbonTable.getDatabaseName))))
var storeLocation = ""
val configuredStore = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
if (null != configuredStore && configuredStore.nonEmpty) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index e7bdff8..07acaa5 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -18,26 +18,23 @@
package org.apache.carbondata.spark.rdd
import java.util
-import java.util.{List, Map}
+import java.util.List
import java.util.concurrent.ExecutorService
import scala.collection.JavaConverters._
-import scala.concurrent.{Await, ExecutionContext, ExecutionContextExecutor, Future}
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.execution.command.{CarbonMergerMapping, CompactionCallableModel, CompactionModel}
-import org.apache.carbondata.core.metadata.PartitionMapFileStore
-import org.apache.carbondata.core.metadata.PartitionMapFileStore.PartitionMapper
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.metadata.SegmentFileStore
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager}
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events._
-import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePostStatusUpdateEvent, LoadTablePreStatusUpdateEvent}
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CompactionType}
import org.apache.carbondata.spark.MergeResultImpl
-import org.apache.carbondata.spark.rdd.CarbonDataRDDFactory.LOGGER
import org.apache.carbondata.spark.util.CommonUtil
/**
@@ -136,39 +133,19 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
var finalMergeStatus = false
val databaseName: String = carbonLoadModel.getDatabaseName
val factTableName = carbonLoadModel.getTableName
- val validSegments: Array[String] = CarbonDataMergerUtil
- .getValidSegments(loadsToMerge).split(',')
- val partitionMapper = if (carbonTable.isHivePartitionTable) {
- var partitionMap: util.Map[String, util.List[String]] = null
- validSegments.foreach { segmentId =>
- val localMapper = new PartitionMapFileStore()
- localMapper.readAllPartitionsOfSegment(
- CarbonTablePath.getSegmentPath(carbonLoadModel.getTablePath, segmentId))
- if (partitionMap == null) {
- partitionMap = localMapper.getPartitionMap
- } else {
- partitionMap.putAll(localMapper.getPartitionMap)
- }
- }
- val mapper = new PartitionMapper()
- mapper.setPartitionMap(partitionMap)
- mapper
- } else {
- null
- }
+ val validSegments: List[Segment] = CarbonDataMergerUtil.getValidSegments(loadsToMerge)
val carbonMergerMapping = CarbonMergerMapping(
tablePath,
carbonTable.getMetaDataFilepath,
mergedLoadName,
databaseName,
factTableName,
- validSegments,
+ validSegments.asScala.toArray,
carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getTableId,
compactionType,
maxSegmentColCardinality = null,
maxSegmentColumnSchemaList = null,
- currentPartitions = partitions,
- partitionMapper)
+ currentPartitions = partitions)
carbonLoadModel.setTablePath(carbonMergerMapping.hdfsStoreLocation)
carbonLoadModel.setLoadMetadataDetails(
SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath).toList.asJava)
@@ -221,11 +198,28 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
if (finalMergeStatus) {
val mergedLoadNumber = CarbonDataMergerUtil.getLoadNumberFromLoadName(mergedLoadName)
- new PartitionMapFileStore().mergePartitionMapFiles(
- CarbonTablePath.getSegmentPath(tablePath, mergedLoadNumber),
- carbonLoadModel.getFactTimeStamp + "")
+ var segmentFileName: String = null
+ if (carbonTable.isHivePartitionTable) {
+ val readPath =
+ CarbonTablePath.getSegmentFilesLocation(carbonLoadModel.getTablePath) +
+ CarbonCommonConstants.FILE_SEPARATOR + carbonLoadModel.getFactTimeStamp + ".tmp"
+ // Merge all partition files into a single file.
+ segmentFileName =
+ mergedLoadNumber + "_" + carbonLoadModel.getFactTimeStamp
+ val segmentFile = SegmentFileStore
+ .mergeSegmentFiles(readPath,
+ segmentFileName,
+ CarbonTablePath.getSegmentFilesLocation(carbonLoadModel.getTablePath))
+ if (segmentFile != null) {
+ SegmentFileStore
+ .moveFromTempFolder(segmentFile,
+ carbonLoadModel.getFactTimeStamp + ".tmp",
+ carbonLoadModel.getTablePath)
+ }
+ segmentFileName = segmentFileName + CarbonTablePath.SEGMENT_EXT
+ }
// trigger event for compaction
- val alterTableCompactionPreStatusUpdateEvent: AlterTableCompactionPreStatusUpdateEvent =
+ val alterTableCompactionPreStatusUpdateEvent =
AlterTableCompactionPreStatusUpdateEvent(sc.sparkSession,
carbonTable,
carbonMergerMapping,
@@ -242,9 +236,13 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
.updateLoadMetadataIUDUpdateDeltaMergeStatus(loadsToMerge,
carbonTable.getMetaDataFilepath,
carbonLoadModel)) ||
- CarbonDataMergerUtil
- .updateLoadMetadataWithMergeStatus(loadsToMerge, carbonTable.getMetaDataFilepath,
- mergedLoadNumber, carbonLoadModel, compactionType)
+ CarbonDataMergerUtil.updateLoadMetadataWithMergeStatus(
+ loadsToMerge,
+ carbonTable.getMetaDataFilepath,
+ mergedLoadNumber,
+ carbonLoadModel,
+ compactionType,
+ segmentFileName)
val compactionLoadStatusPostEvent = AlterTableCompactionPostStatusUpdateEvent(carbonTable,
carbonMergerMapping,
carbonLoadModel,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
index 833c6fe..c286c50 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonCountStar.scala
@@ -55,7 +55,9 @@ case class CarbonCountStar(
CarbonFilters.getPartitions(
Seq.empty,
sparkSession,
- TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName))).asJava),
+ TableIdentifier(
+ carbonTable.getTableName,
+ Some(carbonTable.getDatabaseName))).map(_.asJava).orNull),
absoluteTableIdentifier)
val value = new GenericInternalRow(Seq(Long.box(rowCount)).toArray.asInstanceOf[Array[Any]])
val unsafeProjection = UnsafeProjection.create(output.map(_.dataType).toArray)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 0978fab..46905b8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.CarbonException
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.scan.expression.Expression
@@ -67,7 +68,7 @@ case class CarbonDatasourceHadoopRelation(
def buildScan(requiredColumns: Array[String],
filters: Array[Filter],
- partitions: Seq[String]): RDD[InternalRow] = {
+ partitions: Seq[PartitionSpec]): RDD[InternalRow] = {
val filterExpression: Option[Expression] = filters.flatMap { filter =>
CarbonFilters.createCarbonFilter(schema, filter)
}.reduceOption(new AndExpression(_, _))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index 667d550..7e3b699 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -206,7 +206,9 @@ case class CarbonAlterTableCompactionCommand(
compactionType,
carbonTable,
isCompactionTriggerByDDl,
- CarbonFilters.getCurrentPartitions(sqlContext.sparkSession, carbonTable)
+ CarbonFilters.getCurrentPartitions(sqlContext.sparkSession,
+ TableIdentifier(carbonTable.getTableName,
+ Some(carbonTable.getDatabaseName)))
)
val isConcurrentCompactionAllowed = CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
index 4f90fb5..d2adc57 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.events.{CleanFilesPostEvent, CleanFilesPreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.spark.exception.ConcurrentOperationException
@@ -89,14 +90,10 @@ case class CarbonCleanFilesCommand(
private def cleanGarbageData(sparkSession: SparkSession,
databaseNameOp: Option[String], tableName: String): Unit = {
val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
- val partitions: Option[Seq[String]] = if (carbonTable.isHivePartitionTable) {
- Some(CarbonFilters.getPartitions(
- Seq.empty[Expression],
- sparkSession,
- TableIdentifier(tableName, databaseNameOp)))
- } else {
- None
- }
+ val partitions: Option[Seq[PartitionSpec]] = CarbonFilters.getPartitions(
+ Seq.empty[Expression],
+ sparkSession,
+ TableIdentifier(tableName, databaseNameOp))
CarbonStore.cleanFiles(
dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession),
tableName = tableName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 9bdaddb..7800d3e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -42,6 +42,7 @@ import org.apache.spark.sql.execution.datasources.{CarbonFileFormat, CatalogFile
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.types._
+import org.apache.spark.storage.StorageLevel
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
@@ -51,11 +52,13 @@ import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.dictionary.server.{DictionaryServer, NonSecureDictionaryServer}
import org.apache.carbondata.core.dictionary.service.NonSecureDictionaryServiceProvider
-import org.apache.carbondata.core.metadata.PartitionMapFileStore
+import org.apache.carbondata.core.indexstore.PartitionSpec
+import org.apache.carbondata.core.metadata.SegmentFileStore
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
-import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
+import org.apache.carbondata.core.statusmanager.SegmentStatus
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, DataTypeUtil}
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
@@ -72,7 +75,7 @@ import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServicePr
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.load.DataLoadProcessorStepOnSpark
-import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD}
+import org.apache.carbondata.spark.rdd.CarbonDataRDDFactory
import org.apache.carbondata.spark.util.{CarbonScalaUtil, DataLoadingUtil, GlobalDictionaryUtil, SparkDataTypeConverterImpl}
case class CarbonLoadDataCommand(
@@ -97,6 +100,8 @@ case class CarbonLoadDataCommand(
var sizeInBytes: Long = _
+ var currPartitions: util.List[PartitionSpec] = _
+
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
val dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
@@ -123,6 +128,12 @@ case class CarbonLoadDataCommand(
case l: LogicalRelation => l
}.head
sizeInBytes = logicalPartitionRelation.relation.sizeInBytes
+ currPartitions = CarbonFilters.getCurrentPartitions(
+ sparkSession,
+ TableIdentifier(tableName, databaseNameOp)) match {
+ case Some(parts) => new util.ArrayList(parts.toList.asJava)
+ case _ => null
+ }
}
operationContext.setProperty("isOverwrite", isOverwriteTable)
if(CarbonUtil.hasAggregationDataMap(table)) {
@@ -182,8 +193,9 @@ case class CarbonLoadDataCommand(
options,
optionsFinal,
carbonLoadModel,
- hadoopConf
- )
+ hadoopConf,
+ partition,
+ dataFrame.isDefined)
// Delete stale segment folders that are not in table status but are physically present in
// the Fact folder
LOGGER.info(s"Deleting stale folders if present for table $dbName.$tableName")
@@ -215,7 +227,10 @@ case class CarbonLoadDataCommand(
// First system has to partition the data first and then call the load data
LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
// Clean up the old invalid segment data before creating a new entry for new load.
- DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false, table)
+ DataLoadingUtil.deleteLoadsAndUpdateMetadata(
+ isForceDeletion = false,
+ table,
+ currPartitions)
// add the start entry for the new load in the table status file
if (updateModel.isEmpty && !table.isHivePartitionTable) {
CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(
@@ -259,7 +274,8 @@ case class CarbonLoadDataCommand(
columnar,
partitionStatus,
hadoopConf,
- operationContext)
+ operationContext,
+ LOGGER)
} else {
loadData(
sparkSession,
@@ -267,7 +283,8 @@ case class CarbonLoadDataCommand(
columnar,
partitionStatus,
hadoopConf,
- operationContext)
+ operationContext,
+ LOGGER)
}
val loadTablePostExecutionEvent: LoadTablePostExecutionEvent =
new LoadTablePostExecutionEvent(
@@ -331,7 +348,9 @@ case class CarbonLoadDataCommand(
columnar: Boolean,
partitionStatus: SegmentStatus,
hadoopConf: Configuration,
- operationContext: OperationContext): Unit = {
+ operationContext: OperationContext,
+ LOGGER: LogService): Seq[Row] = {
+ var rows = Seq.empty[Row]
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val carbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier
.getCarbonTableIdentifier
@@ -418,12 +437,13 @@ case class CarbonLoadDataCommand(
if (carbonTable.isHivePartitionTable) {
try {
- loadDataWithPartition(
+ rows = loadDataWithPartition(
sparkSession,
carbonLoadModel,
hadoopConf,
loadDataFrame,
- operationContext)
+ operationContext,
+ LOGGER)
} finally {
server match {
case Some(dictServer) =>
@@ -450,6 +470,7 @@ case class CarbonLoadDataCommand(
updateModel,
operationContext)
}
+ rows
}
private def loadData(
@@ -458,7 +479,9 @@ case class CarbonLoadDataCommand(
columnar: Boolean,
partitionStatus: SegmentStatus,
hadoopConf: Configuration,
- operationContext: OperationContext): Unit = {
+ operationContext: OperationContext,
+ LOGGER: LogService): Seq[Row] = {
+ var rows = Seq.empty[Row]
val (dictionaryDataFrame, loadDataFrame) = if (updateModel.isDefined) {
val dataFrameWithTupleId: DataFrame = getDataFrameWithTupleID()
// getting all fields except tupleId field as it is not required in the value
@@ -478,12 +501,12 @@ case class CarbonLoadDataCommand(
dictionaryDataFrame)
}
if (table.isHivePartitionTable) {
- loadDataWithPartition(
+ rows = loadDataWithPartition(
sparkSession,
carbonLoadModel,
hadoopConf,
loadDataFrame,
- operationContext)
+ operationContext, LOGGER)
} else {
CarbonDataRDDFactory.loadCarbonData(
sparkSession.sqlContext,
@@ -497,6 +520,7 @@ case class CarbonLoadDataCommand(
updateModel,
operationContext)
}
+ rows
}
/**
@@ -504,24 +528,16 @@ case class CarbonLoadDataCommand(
* into partitoned data. The table relation would be converted to HadoopFSRelation to let spark
* handling the partitioning.
*/
- private def loadDataWithPartition(sparkSession: SparkSession,
+ private def loadDataWithPartition(
+ sparkSession: SparkSession,
carbonLoadModel: CarbonLoadModel,
hadoopConf: Configuration,
dataFrame: Option[DataFrame],
- operationContext: OperationContext): Unit = {
+ operationContext: OperationContext,
+ LOGGER: LogService): Seq[Row] = {
val table = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val identifier = TableIdentifier(table.getTableName, Some(table.getDatabaseName))
val catalogTable: CatalogTable = logicalPartitionRelation.catalogTable.get
- val currentPartitions =
- CarbonFilters.getPartitions(Seq.empty[Expression], sparkSession, identifier)
- // Clean up the alreday dropped partitioned data
- new PartitionMapFileStore().cleanSegments(table, currentPartitions.asJava, false)
- // Converts the data to carbon understandable format. The timestamp/date format data needs to
- // converted to hive standard fomat to let spark understand the data to partition.
- val serializationNullFormat =
- carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
- val badRecordAction =
- carbonLoadModel.getBadRecordsAction.split(",")(1)
var timeStampformatString = carbonLoadModel.getTimestampformat
if (timeStampformatString.isEmpty) {
timeStampformatString = carbonLoadModel.getDefaultTimestampFormat
@@ -532,127 +548,114 @@ case class CarbonLoadDataCommand(
dateFormatString = carbonLoadModel.getDefaultDateFormat
}
val dateFormat = new SimpleDateFormat(dateFormatString)
- CarbonSession.threadSet(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT, dateFormatString)
- CarbonSession.threadSet(
- CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT,
- timeStampformatString)
- CarbonSession.threadSet(
- CarbonLoadOptionConstants.CARBON_OPTIONS_SERIALIZATION_NULL_FORMAT,
- serializationNullFormat)
- CarbonSession.threadSet(
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
- badRecordAction)
- val isEmptyBadRecord = carbonLoadModel.getIsEmptyDataBadRecord.split(",")(1)
- CarbonSession.threadSet(
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
- isEmptyBadRecord)
+ // Clean up the alreday dropped partitioned data
+ SegmentFileStore.cleanSegments(table, null, false)
CarbonSession.threadSet("partition.operationcontext", operationContext)
// input data from csv files. Convert to logical plan
val allCols = new ArrayBuffer[String]()
allCols ++= table.getAllDimensions.asScala.map(_.getColName)
allCols ++= table.getAllMeasures.asScala.map(_.getColName)
var attributes =
- StructType(allCols.map(StructField(_, StringType))).toAttributes
+ StructType(
+ allCols.filterNot(_.equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)).map(
+ StructField(_, StringType))).toAttributes
var partitionsLen = 0
val sortScope = CarbonDataProcessorUtil.getSortScope(carbonLoadModel.getSortScope)
- def transformQuery(rdd: RDD[Row], isDataFrame: Boolean) = {
- val updatedRdd = convertData(rdd, sparkSession, carbonLoadModel, isDataFrame)
- val catalogAttributes = catalogTable.schema.toAttributes
- attributes = attributes.map(a => {
- catalogAttributes.find(_.name.equalsIgnoreCase(a.name)).get
- })
- attributes = attributes.map { attr =>
- val column = table.getColumnByName(table.getTableName, attr.name)
- if (column.hasEncoding(Encoding.DICTIONARY)) {
- AttributeReference(
- attr.name,
- IntegerType,
- attr.nullable,
- attr.metadata)(attr.exprId, attr.qualifier, attr.isGenerated)
- } else if (attr.dataType == TimestampType || attr.dataType == DateType) {
- AttributeReference(
- attr.name,
- LongType,
- attr.nullable,
- attr.metadata)(attr.exprId, attr.qualifier, attr.isGenerated)
- } else {
- attr
- }
- }
- // Only select the required columns
- val output = if (partition.nonEmpty) {
- val lowerCasePartition = partition.map { case (key, value) => (key.toLowerCase, value) }
- catalogTable.schema.map { attr =>
- attributes.find(_.name.equalsIgnoreCase(attr.name)).get
- }.filter(attr => lowerCasePartition.getOrElse(attr.name.toLowerCase, None).isEmpty)
- } else {
- catalogTable.schema.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get)
- }
- partitionsLen = rdd.partitions.length
- val child = Project(output, LogicalRDD(attributes, updatedRdd)(sparkSession))
- if (sortScope == SortScopeOptions.SortScope.GLOBAL_SORT) {
- val sortColumns = table.getSortColumns(table.getTableName)
- Sort(output.filter(f => sortColumns.contains(f.name)).map(SortOrder(_, Ascending)),
- true,
- child)
- } else {
- child
- }
+ val partitionValues = if (partition.nonEmpty) {
+ partition.filter(_._2.nonEmpty).map{ case(col, value) =>
+ val field = catalogTable.schema.find(_.name.equalsIgnoreCase(col)).get
+ CarbonScalaUtil.convertToDateAndTimeFormats(
+ value.get,
+ field.dataType,
+ timeStampFormat,
+ dateFormat)
+ }.toArray
+ } else {
+ Array[String]()
}
-
+ var persistedRDD: Option[RDD[InternalRow]] = None
try {
val query: LogicalPlan = if (dataFrame.isDefined) {
- val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
- val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
- val dfAttributes =
- StructType(dataFrame.get.schema.fields.map(_.copy(dataType = StringType))).toAttributes
- val partitionValues = if (partition.nonEmpty) {
- partition.values.filter(_.nonEmpty).map(_.get).toArray
+ val (rdd, dfAttributes) = if (updateModel.isDefined) {
+ // Get the updated query plan in case of update scenario
+ val updatedFrame = Dataset.ofRows(
+ sparkSession,
+ getLogicalQueryForUpdate(
+ sparkSession,
+ catalogTable,
+ dataFrame.get,
+ carbonLoadModel))
+ (updatedFrame.rdd, updatedFrame.schema)
} else {
- Array[String]()
+ if (partition.nonEmpty) {
+ val headers = carbonLoadModel.getCsvHeaderColumns.dropRight(partition.size)
+ val updatedHeader = headers ++ partition.keys.map(_.toLowerCase)
+ carbonLoadModel.setCsvHeader(updatedHeader.mkString(","))
+ carbonLoadModel.setCsvHeaderColumns(carbonLoadModel.getCsvHeader.split(","))
+ }
+ (dataFrame.get.rdd, dataFrame.get.schema)
+ }
+
+ val expectedColumns = {
+ val staticPartCols = partition.filter(_._2.isDefined).keySet
+ attributes.filterNot(a => staticPartCols.contains(a.name))
+ }
+ if (expectedColumns.length != dfAttributes.length) {
+ throw new AnalysisException(
+ s"Cannot insert into table $tableName because the number of columns are different: " +
+ s"need ${expectedColumns.length} columns, " +
+ s"but query has ${dfAttributes.length} columns.")
+ }
+ val nonPartitionBounds = expectedColumns.zipWithIndex.map(_._2).toArray
+ val partitionBounds = new Array[Int](partitionValues.length)
+ if (partition.nonEmpty) {
+ val nonPartitionSchemaLen = attributes.length - partition.size
+ var i = nonPartitionSchemaLen
+ var index = 0
+ var partIndex = 0
+ partition.values.foreach { p =>
+ if (p.isDefined) {
+ partitionBounds(partIndex) = nonPartitionSchemaLen + index
+ partIndex = partIndex + 1
+ } else {
+ nonPartitionBounds(i) = nonPartitionSchemaLen + index
+ i = i + 1
+ }
+ index = index + 1
+ }
}
- val len = dfAttributes.length
- val rdd = dataFrame.get.rdd.map { f =>
+
+ val len = dfAttributes.length + partitionValues.length
+ val transRdd = rdd.map { f =>
val data = new Array[Any](len)
var i = 0
while (i < f.length) {
- data(i) =
- UTF8String.fromString(
- CarbonScalaUtil.getString(f.get(i),
- serializationNullFormat,
- delimiterLevel1,
- delimiterLevel2,
- timeStampFormat,
- dateFormat))
+ data(nonPartitionBounds(i)) = f.get(i)
i = i + 1
}
- if (partitionValues.length > 0) {
- var j = 0
- while (i < len) {
- data(i) = UTF8String.fromString(partitionValues(j))
- j = j + 1
- i = i + 1
- }
+ var j = 0
+ while (j < partitionBounds.length) {
+ data(partitionBounds(j)) = UTF8String.fromString(partitionValues(j))
+ j = j + 1
}
Row.fromSeq(data)
}
- val transRdd = if (updateModel.isDefined) {
- // Get the updated query plan in case of update scenario
- Dataset.ofRows(
+
+ val (transformedPlan, partitions, persistedRDDLocal) =
+ transformQuery(
+ transRdd,
sparkSession,
- getLogicalQueryForUpdate(
- sparkSession,
- catalogTable,
- dfAttributes,
- rdd.map(row => InternalRow.fromSeq(row.toSeq)),
- carbonLoadModel)).rdd
- } else {
- rdd
- }
- transformQuery(transRdd, true)
+ carbonLoadModel,
+ partitionValues,
+ catalogTable,
+ attributes,
+ sortScope,
+ isDataFrame = true)
+ partitionsLen = partitions
+ persistedRDD = persistedRDDLocal
+ transformedPlan
} else {
-
val rowDataTypes = attributes.map { attribute =>
catalogTable.schema.find(_.name.equalsIgnoreCase(attribute.name)) match {
case Some(attr) => attr.dataType
@@ -667,12 +670,29 @@ case class CarbonLoadDataCommand(
}
}
val columnCount = carbonLoadModel.getCsvHeaderColumns.length
- var rdd = DataLoadingUtil.csvFileScanRDD(
+ val rdd = DataLoadingUtil.csvFileScanRDD(
sparkSession,
model = carbonLoadModel,
hadoopConf).map(DataLoadProcessorStepOnSpark.toStringArrayRow(_, columnCount))
- transformQuery(rdd.asInstanceOf[RDD[Row]], false)
+ val (transformedPlan, partitions, persistedRDDLocal) =
+ transformQuery(
+ rdd.asInstanceOf[RDD[Row]],
+ sparkSession,
+ carbonLoadModel,
+ partitionValues,
+ catalogTable,
+ attributes,
+ sortScope,
+ isDataFrame = false)
+ partitionsLen = partitions
+ persistedRDD = persistedRDDLocal
+ transformedPlan
+ }
+ if (updateModel.isDefined) {
+ carbonLoadModel.setFactTimeStamp(updateModel.get.updatedTimeStamp)
}
+ // Create and ddd the segment to the tablestatus.
+ CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(carbonLoadModel, isOverwriteTable)
val convertRelation = convertToLogicalRelation(
catalogTable,
sizeInBytes,
@@ -703,23 +723,37 @@ case class CarbonLoadDataCommand(
overwrite = false,
ifPartitionNotExists = false)
Dataset.ofRows(sparkSession, convertedPlan)
+ } catch {
+ case ex: Throwable =>
+ val (executorMessage, errorMessage) = CarbonScalaUtil.retrieveAndLogErrorMsg(ex, LOGGER)
+ if (updateModel.isDefined) {
+ CarbonScalaUtil.updateErrorInUpdateModel(updateModel.get, executorMessage)
+ }
+ LOGGER.info(errorMessage)
+ LOGGER.error(ex)
+ throw new Exception(errorMessage)
} finally {
- CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
- CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT)
- CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_SERIALIZATION_NULL_FORMAT)
- CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
- CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
CarbonSession.threadUnset("partition.operationcontext")
if (isOverwriteTable) {
DataMapStoreManager.getInstance().clearDataMaps(table.getAbsoluteTableIdentifier)
// Clean the overwriting segments if any.
- new PartitionMapFileStore().cleanSegments(
+ SegmentFileStore.cleanSegments(
table,
- CarbonFilters.getPartitions(Seq.empty, sparkSession, identifier).asJava,
+ null,
false)
}
+ if (partitionsLen > 1) {
+ // clean cache only if persisted and keeping unpersist non-blocking as non-blocking call
+ // will not have any functional impact as spark automatically monitors the cache usage on
+ // each node and drops out old data partitions in a least-recently used (LRU) fashion.
+ persistedRDD match {
+ case Some(rdd) => rdd.unpersist(false)
+ case _ =>
+ }
+ }
}
try {
+ carbonLoadModel.setFactTimeStamp(System.currentTimeMillis())
// Trigger auto compaction
CarbonDataRDDFactory.handleSegmentMerging(
sparkSession.sqlContext,
@@ -732,37 +766,140 @@ case class CarbonLoadDataCommand(
"Dataload is success. Auto-Compaction has failed. Please check logs.",
e)
}
+ val specs =
+ SegmentFileStore.getPartitionSpecs(carbonLoadModel.getSegmentId, carbonLoadModel.getTablePath)
+ if (specs != null) {
+ specs.asScala.map{ spec =>
+ Row(spec.getPartitions.asScala.mkString("/"), spec.getLocation.toString, spec.getUuid)
+ }
+ } else {
+ Seq.empty[Row]
+ }
}
+ /**
+ * Transform the rdd to logical plan as per the sortscope. If it is global sort scope then it
+ * will convert to sort logical plan otherwise project plan.
+ */
+ private def transformQuery(rdd: RDD[Row],
+ sparkSession: SparkSession,
+ loadModel: CarbonLoadModel,
+ partitionValues: Array[String],
+ catalogTable: CatalogTable,
+ curAttributes: Seq[AttributeReference],
+ sortScope: SortScopeOptions.SortScope,
+ isDataFrame: Boolean): (LogicalPlan, Int, Option[RDD[InternalRow]]) = {
+ // Converts the data as per the loading steps before give it to writer or sorter
+ val updatedRdd = convertData(
+ rdd,
+ sparkSession,
+ loadModel,
+ isDataFrame,
+ partitionValues)
+ val catalogAttributes = catalogTable.schema.toAttributes
+ var attributes = curAttributes.map(a => {
+ catalogAttributes.find(_.name.equalsIgnoreCase(a.name)).get
+ })
+ attributes = attributes.map { attr =>
+ // Update attribute datatypes in case of dictionary columns, in case of dictionary columns
+ // datatype is always int
+ val column = table.getColumnByName(table.getTableName, attr.name)
+ if (column.hasEncoding(Encoding.DICTIONARY)) {
+ AttributeReference(
+ attr.name,
+ IntegerType,
+ attr.nullable,
+ attr.metadata)(attr.exprId, attr.qualifier, attr.isGenerated)
+ } else if (attr.dataType == TimestampType || attr.dataType == DateType) {
+ AttributeReference(
+ attr.name,
+ LongType,
+ attr.nullable,
+ attr.metadata)(attr.exprId, attr.qualifier, attr.isGenerated)
+ } else {
+ attr
+ }
+ }
+ // Only select the required columns
+ val output = if (partition.nonEmpty) {
+ val lowerCasePartition = partition.map { case (key, value) => (key.toLowerCase, value) }
+ catalogTable.schema.map { attr =>
+ attributes.find(_.name.equalsIgnoreCase(attr.name)).get
+ }.filter(attr => lowerCasePartition.getOrElse(attr.name.toLowerCase, None).isEmpty)
+ } else {
+ catalogTable.schema.map(f => attributes.find(_.name.equalsIgnoreCase(f.name)).get)
+ }
+ val partitionsLen = rdd.partitions.length
+
+ // If it is global sort scope then appl sort logical plan on the sort columns
+ if (sortScope == SortScopeOptions.SortScope.GLOBAL_SORT) {
+ // Because if the number of partitions greater than 1, there will be action operator(sample)
+ // in sortBy operator. So here we cache the rdd to avoid do input and convert again.
+ if (partitionsLen > 1) {
+ updatedRdd.persist(StorageLevel.fromString(
+ CarbonProperties.getInstance().getGlobalSortRddStorageLevel))
+ }
+ val child = Project(output, LogicalRDD(attributes, updatedRdd)(sparkSession))
+ val sortColumns = table.getSortColumns(table.getTableName)
+ val sortPlan =
+ Sort(
+ output.filter(f => sortColumns.contains(f.name)).map(SortOrder(_, Ascending)),
+ global = true,
+ child)
+ (sortPlan, partitionsLen, Some(updatedRdd))
+ } else {
+ (Project(output, LogicalRDD(attributes, updatedRdd)(sparkSession)), partitionsLen, None)
+ }
+ }
+
+ /**
+ * Convert the rdd as per steps of data loading inputprocessor step and coverter step
+ * @param originRDD
+ * @param sparkSession
+ * @param model
+ * @param isDataFrame
+ * @param partitionValues
+ * @return
+ */
private def convertData(
originRDD: RDD[Row],
sparkSession: SparkSession,
model: CarbonLoadModel,
- isDataFrame: Boolean): RDD[InternalRow] = {
+ isDataFrame: Boolean,
+ partitionValues: Array[String]): RDD[InternalRow] = {
model.setPartitionId("0")
val sc = sparkSession.sparkContext
+ val info =
+ model.getCarbonDataLoadSchema.getCarbonTable.getTableInfo.getFactTable.getPartitionInfo
+ info.setColumnSchemaList(new util.ArrayList[ColumnSchema](info.getColumnSchemaList))
val modelBroadcast = sc.broadcast(model)
val partialSuccessAccum = sc.accumulator(0, "Partial Success Accumulator")
val inputStepRowCounter = sc.accumulator(0, "Input Processor Accumulator")
// 1. Input
- var convertRDD =
+ val convertRDD =
if (isDataFrame) {
originRDD.mapPartitions{rows =>
DataLoadProcessorStepOnSpark.toRDDIterator(rows, modelBroadcast)
}
} else {
- originRDD.map{row =>
- val array = new Array[AnyRef](row.length)
+ // Append the partition columns in case of static partition scenario
+ val partitionLen = partitionValues.length
+ val len = model.getCsvHeaderColumns.length - partitionLen
+ originRDD.map{ row =>
+ val array = new Array[AnyRef](len + partitionLen)
var i = 0
- while (i < array.length) {
+ while (i < len) {
array(i) = row.get(i).asInstanceOf[AnyRef]
i = i + 1
}
+ if (partitionLen > 0) {
+ System.arraycopy(partitionValues, 0, array, i, partitionLen)
+ }
array
}
}
- val finalRDD = convertRDD.mapPartitionsWithIndex { case (index, rows) =>
+ val finalRDD = convertRDD.mapPartitionsWithIndex {case(index, rows) =>
DataTypeUtil.setDataTypeConverter(new SparkDataTypeConverterImpl)
DataLoadProcessorStepOnSpark.inputAndconvertFunc(
rows,
@@ -783,12 +920,11 @@ case class CarbonLoadDataCommand(
private def getLogicalQueryForUpdate(
sparkSession: SparkSession,
catalogTable: CatalogTable,
- attributes: Seq[AttributeReference],
- rdd: RDD[InternalRow],
+ df: DataFrame,
carbonLoadModel: CarbonLoadModel): LogicalPlan = {
sparkSession.sparkContext.setLocalProperty(EXECUTION_ID_KEY, null)
// In case of update, we don't need the segmrntid column in case of partitioning
- val dropAttributes = attributes.dropRight(1)
+ val dropAttributes = df.logicalPlan.output.dropRight(1)
val finalOutput = catalogTable.schema.map { attr =>
dropAttributes.find { d =>
val index = d.name.lastIndexOf("-updatedColumn")
@@ -801,7 +937,7 @@ case class CarbonLoadDataCommand(
}
carbonLoadModel.setCsvHeader(catalogTable.schema.map(_.name.toLowerCase).mkString(","))
carbonLoadModel.setCsvHeaderColumns(carbonLoadModel.getCsvHeader.split(","))
- Project(finalOutput, LogicalRDD(attributes, rdd)(sparkSession))
+ Project(finalOutput, df.logicalPlan)
}
private def convertToLogicalRelation(
@@ -855,9 +991,19 @@ case class CarbonLoadDataCommand(
if (updateModel.isDefined) {
options += (("updatetimestamp", updateModel.get.updatedTimeStamp.toString))
if (updateModel.get.deletedSegments.nonEmpty) {
- options += (("segmentsToBeDeleted", updateModel.get.deletedSegments.mkString(",")))
+ options += (("segmentsToBeDeleted",
+ updateModel.get.deletedSegments.map(_.getSegmentNo).mkString(",")))
}
}
+ if (currPartitions != null) {
+ val currPartStr = ObjectSerializationUtil.convertObjectToString(currPartitions)
+ options += (("currentpartition", currPartStr))
+ }
+ if (loadModel.getSegmentId != null) {
+ val currLoadEntry =
+ ObjectSerializationUtil.convertObjectToString(loadModel.getCurrentLoadMetadataDetail)
+ options += (("currentloadentry", currLoadEntry))
+ }
val hdfsRelation = HadoopFsRelation(
location = catalog,
partitionSchema = partitionSchema,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
index 72ed051..e3e4c7a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
@@ -28,8 +28,10 @@ import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, Me
import org.apache.spark.sql.execution.command.table.CarbonCreateTableCommand
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier, PartitionMapFileStore}
+import org.apache.carbondata.core.indexstore.PartitionSpec
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier, SegmentFileStore}
import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.{DataMapSchema, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
@@ -228,10 +230,17 @@ case class RefreshCarbonTableCommand(
val allpartitions = metadataDetails.map{ metadata =>
if (metadata.getSegmentStatus == SegmentStatus.SUCCESS ||
metadata.getSegmentStatus == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
- val mapper = new PartitionMapFileStore()
- mapper.readAllPartitionsOfSegment(
- CarbonTablePath.getSegmentPath(absIdentifier.getTablePath, metadata.getLoadName))
- Some(mapper.getPartitionMap.values().asScala)
+ val mapper = new SegmentFileStore(absIdentifier.getTablePath, metadata.getSegmentFile)
+ val specs = mapper.getLocationMap.asScala.map { case(location, fd) =>
+ var updatedLoc =
+ if (fd.isRelative) {
+ absIdentifier.getTablePath + CarbonCommonConstants.FILE_SEPARATOR + location
+ } else {
+ location
+ }
+ new PartitionSpec(fd.getPartitions, updatedLoc)
+ }
+ Some(specs)
} else {
None
}
@@ -240,14 +249,14 @@ case class RefreshCarbonTableCommand(
TableIdentifier(absIdentifier.getTableName, Some(absIdentifier.getDatabaseName))
// Register the partition information to the hive metastore
allpartitions.foreach { segPartitions =>
- val specs: Seq[TablePartitionSpec] = segPartitions.map { indexPartitions =>
- indexPartitions.asScala.map{ p =>
+ val specs: Seq[(TablePartitionSpec, Option[String])] = segPartitions.map { indexPartitions =>
+ (indexPartitions.getPartitions.asScala.map{ p =>
val spec = p.split("=")
(spec(0), spec(1))
- }.toMap
+ }.toMap, Some(indexPartitions.getLocation.toString))
}.toSeq
// Add partition information
- AlterTableAddPartitionCommand(identifier, specs.map((_, None)), true).run(sparkSession)
+ AlterTableAddPartitionCommand(identifier, specs, true).run(sparkSession)
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
index 756d120..4886676 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
@@ -27,6 +27,7 @@ import org.apache.spark.storage.StorageLevel
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
@@ -173,7 +174,7 @@ private[sql] case class CarbonProjectForUpdateCommand(
sparkSession: SparkSession,
currentTime: Long,
executorErrors: ExecutionErrors,
- deletedSegments: Seq[String]): Unit = {
+ deletedSegments: Seq[Segment]): Unit = {
def isDestinationRelation(relation: CarbonDatasourceHadoopRelation): Boolean = {
val dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
index 1ac0b34..25d5e91 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
@@ -35,6 +35,7 @@ import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, DeleteDeltaBlockDetails, SegmentUpdateDetails, TupleIdEnum}
@@ -61,15 +62,20 @@ object DeleteExecution {
dataRdd: RDD[Row],
timestamp: String,
isUpdateOperation: Boolean,
- executorErrors: ExecutionErrors): Seq[String] = {
+ executorErrors: ExecutionErrors): Seq[Segment] = {
var res: Array[List[(SegmentStatus, (SegmentUpdateDetails, ExecutionErrors))]] = null
val database = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
val carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier)
- val factPath = carbonTablePath.getFactDir
- var segmentsTobeDeleted = Seq.empty[String]
+ val isPartitionTable = carbonTable.isHivePartitionTable
+ val factPath = if (isPartitionTable) {
+ carbonTablePath.getPath
+ } else {
+ carbonTablePath.getFactDir
+ }
+ var segmentsTobeDeleted = Seq.empty[Segment]
val deleteRdd = if (isUpdateOperation) {
val schema =
@@ -104,7 +110,7 @@ object DeleteExecution {
CarbonFilters.getPartitions(
Seq.empty,
sparkSession,
- TableIdentifier(tableName, databaseNameOp)).asJava)
+ TableIdentifier(tableName, databaseNameOp)).map(_.asJava).orNull)
val segmentUpdateStatusMngr = new SegmentUpdateStatusManager(absoluteTableIdentifier)
CarbonUpdateUtil
.createBlockDetailsMap(blockMappingVO, segmentUpdateStatusMngr)
@@ -144,12 +150,12 @@ object DeleteExecution {
// all or none : update status file, only if complete delete opeartion is successfull.
def checkAndUpdateStatusFiles(): Unit = {
val blockUpdateDetailsList = new util.ArrayList[SegmentUpdateDetails]()
- val segmentDetails = new util.HashSet[String]()
+ val segmentDetails = new util.HashSet[Segment]()
res.foreach(resultOfSeg => resultOfSeg.foreach(
resultOfBlock => {
if (resultOfBlock._1 == SegmentStatus.SUCCESS) {
blockUpdateDetailsList.add(resultOfBlock._2._1)
- segmentDetails.add(resultOfBlock._2._1.getSegmentName)
+ segmentDetails.add(new Segment(resultOfBlock._2._1.getSegmentName, null))
// if this block is invalid then decrement block count in map.
if (CarbonUpdateUtil.isBlockInvalid(resultOfBlock._2._1.getSegmentStatus)) {
CarbonUpdateUtil.decrementDeletedBlockCount(resultOfBlock._2._1,
@@ -250,7 +256,7 @@ object DeleteExecution {
countOfRows = countOfRows + 1
}
- val blockPath = CarbonUpdateUtil.getTableBlockPath(TID, factPath)
+ val blockPath = CarbonUpdateUtil.getTableBlockPath(TID, factPath, isPartitionTable)
val completeBlockName = CarbonTablePath
.addDataPartPrefix(CarbonUpdateUtil.getRequiredFieldFromTID(TID, TupleIdEnum.BLOCK_ID) +
CarbonCommonConstants.FACT_FILE_EXT)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala
index bdecac1..f88e767 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/HorizontalCompaction.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.execution.command.management.CarbonAlterTableCompact
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager
@@ -113,7 +114,7 @@ object HorizontalCompaction {
absTableIdentifier: AbsoluteTableIdentifier,
segmentUpdateStatusManager: SegmentUpdateStatusManager,
factTimeStamp: Long,
- segLists: util.List[String]): Unit = {
+ segLists: util.List[Segment]): Unit = {
val db = carbonTable.getDatabaseName
val table = carbonTable.getTableName
// get the valid segments qualified for update compaction.
@@ -163,7 +164,7 @@ object HorizontalCompaction {
absTableIdentifier: AbsoluteTableIdentifier,
segmentUpdateStatusManager: SegmentUpdateStatusManager,
factTimeStamp: Long,
- segLists: util.List[String]): Unit = {
+ segLists: util.List[Segment]): Unit = {
val db = carbonTable.getDatabaseName
val table = carbonTable.getTableName
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableAddHivePartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableAddHivePartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableAddHivePartitionCommand.scala
new file mode 100644
index 0000000..2aaecc7
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableAddHivePartitionCommand.scala
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.partition
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, AlterTableDropPartitionCommand, AtomicRunnableCommand}
+import org.apache.spark.sql.optimizer.CarbonFilters
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.indexstore.PartitionSpec
+import org.apache.carbondata.core.metadata.SegmentFileStore
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.statusmanager.SegmentStatus
+import org.apache.carbondata.core.util.CarbonUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
+import org.apache.carbondata.processing.util.CarbonLoaderUtil
+
+/**
+ * Adding the partition to the hive and create a new segment if the location has data.
+ *
+ */
+case class CarbonAlterTableAddHivePartitionCommand(
+ tableName: TableIdentifier,
+ partitionSpecsAndLocs: Seq[(TablePartitionSpec, Option[String])],
+ ifNotExists: Boolean)
+ extends AtomicRunnableCommand {
+
+ var partitionSpecsAndLocsTobeAdded : util.List[PartitionSpec] = _
+ var table: CarbonTable = _
+
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
+ table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ if (table.isHivePartitionTable) {
+ val partitionWithLoc = partitionSpecsAndLocs.filter(_._2.isDefined)
+ if (partitionWithLoc.nonEmpty) {
+ val partitionSpecs = partitionWithLoc.map{ case (part, location) =>
+ new PartitionSpec(
+ new util.ArrayList(part.map(p => p._1 + "=" + p._2).toList.asJava),
+ location.get)
+ }
+ // Get all the partitions which are not already present in hive.
+ val currParts = CarbonFilters.getCurrentPartitions(sparkSession, tableName).get
+ partitionSpecsAndLocsTobeAdded =
+ new util.ArrayList(partitionSpecs.filterNot { part =>
+ currParts.exists(p => part.equals(p))
+ }.asJava)
+ }
+ AlterTableAddPartitionCommand(tableName, partitionSpecsAndLocs, ifNotExists).run(sparkSession)
+ }
+ Seq.empty[Row]
+ }
+
+
+ override def undoMetadata(sparkSession: SparkSession, exception: Exception): Seq[Row] = {
+ AlterTableDropPartitionCommand(tableName, partitionSpecsAndLocs.map(_._1), true, false, true)
+ val msg = s"Got exception $exception when processing data of add partition." +
+ "Dropping partitions to the metadata"
+ LogServiceFactory.getLogService(this.getClass.getCanonicalName).error(msg)
+ Seq.empty[Row]
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ // Partitions with physical data should be registered to as a new segment.
+ if (partitionSpecsAndLocsTobeAdded != null && partitionSpecsAndLocsTobeAdded.size() > 0) {
+ val segmentFile = SegmentFileStore.getSegmentFileForPhysicalDataPartitions(table.getTablePath,
+ partitionSpecsAndLocsTobeAdded)
+ if (segmentFile != null) {
+ val loadModel = new CarbonLoadModel
+ loadModel.setCarbonDataLoadSchema(new CarbonDataLoadSchema(table))
+ // Create new entry in tablestatus file
+ CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(loadModel, false)
+ val newMetaEntry = loadModel.getCurrentLoadMetadataDetail
+ val segmentFileName =
+ loadModel.getSegmentId + "_" + loadModel.getFactTimeStamp + CarbonTablePath.SEGMENT_EXT
+ newMetaEntry.setSegmentFile(segmentFileName)
+ val segmentsLoc = CarbonTablePath.getSegmentFilesLocation(table.getTablePath)
+ CarbonUtil.checkAndCreateFolder(segmentsLoc)
+ val segmentPath = segmentsLoc + CarbonCommonConstants.FILE_SEPARATOR + segmentFileName
+ SegmentFileStore.writeSegmentFile(segmentFile, segmentPath)
+ CarbonLoaderUtil.populateNewLoadMetaEntry(
+ newMetaEntry,
+ SegmentStatus.SUCCESS,
+ loadModel.getFactTimeStamp,
+ true)
+ // Add size to the entry
+ CarbonLoaderUtil.addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId, table)
+ // Make the load as success in table status
+ CarbonLoaderUtil.recordNewLoadMetadata(newMetaEntry, loadModel, false, false)
+ }
+ }
+ Seq.empty[Row]
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
index cb4dece..407057e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropHivePartitionCommand.scala
@@ -25,17 +25,17 @@ import org.apache.spark.sql.{AnalysisException, CarbonEnv, Row, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.execution.command.{AlterTableAddPartitionCommand, AlterTableDropPartitionCommand, AtomicRunnableCommand}
-import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.util.AlterTableUtil
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
-import org.apache.carbondata.core.metadata.PartitionMapFileStore
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil
+import org.apache.carbondata.core.metadata.SegmentFileStore
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonUtil
-import org.apache.carbondata.spark.rdd.{CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD}
+import org.apache.carbondata.spark.rdd.CarbonDropPartitionRDD
/**
* Drop the partitions from hive and carbon store. It drops the partitions in following steps
@@ -59,16 +59,43 @@ case class CarbonAlterTableDropHivePartitionCommand(
retainData: Boolean)
extends AtomicRunnableCommand {
+ var carbonPartitionsTobeDropped : util.List[PartitionSpec] = _
+ var table: CarbonTable = _
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
- val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
if (CarbonUtil.hasAggregationDataMap(table)) {
throw new AnalysisException(
"Partition can not be dropped as it is mapped to Pre Aggregate table")
}
if (table.isHivePartitionTable) {
+ var locks = List.empty[ICarbonLock]
try {
- specs.flatMap(f => sparkSession.sessionState.catalog.listPartitions(tableName, Some(f)))
+ val locksToBeAcquired = List(LockUsage.METADATA_LOCK,
+ LockUsage.COMPACTION_LOCK,
+ LockUsage.DELETE_SEGMENT_LOCK,
+ LockUsage.DROP_TABLE_LOCK,
+ LockUsage.CLEAN_FILES_LOCK,
+ LockUsage.ALTER_PARTITION_LOCK)
+ locks = AlterTableUtil.validateTableAndAcquireLock(
+ table.getDatabaseName,
+ table.getTableName,
+ locksToBeAcquired)(sparkSession)
+ val partitions =
+ specs.flatMap(f => sparkSession.sessionState.catalog.listPartitions(tableName, Some(f)))
+ val carbonPartitions = partitions.map { partition =>
+ new PartitionSpec(new util.ArrayList[String](
+ partition.spec.seq.map { case (column, value) => column + "=" + value }.toList.asJava),
+ partition.location)
+ }
+ carbonPartitionsTobeDropped = new util.ArrayList[PartitionSpec](carbonPartitions.asJava)
+ // Drop the partitions from hive.
+ AlterTableDropPartitionCommand(
+ tableName,
+ specs,
+ ifExists,
+ purge,
+ retainData).run(sparkSession)
} catch {
case e: Exception =>
if (!ifExists) {
@@ -77,15 +104,10 @@ case class CarbonAlterTableDropHivePartitionCommand(
log.warn(e.getMessage)
return Seq.empty[Row]
}
+ } finally {
+ AlterTableUtil.releaseLocks(locks)
}
- // Drop the partitions from hive.
- AlterTableDropPartitionCommand(
- tableName,
- specs,
- ifExists,
- purge,
- retainData).run(sparkSession)
}
Seq.empty[Row]
}
@@ -100,7 +122,6 @@ case class CarbonAlterTableDropHivePartitionCommand(
}
override def processData(sparkSession: SparkSession): Seq[Row] = {
- val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
var locks = List.empty[ICarbonLock]
val uniqueId = System.currentTimeMillis().toString
try {
@@ -119,48 +140,27 @@ case class CarbonAlterTableDropHivePartitionCommand(
}.toSet
val segments = new SegmentStatusManager(table.getAbsoluteTableIdentifier)
.getValidAndInvalidSegments.getValidSegments
- try {
- // First drop the partitions from partition mapper files of each segment
- new CarbonDropPartitionRDD(sparkSession.sparkContext,
- table.getTablePath,
- segments.asScala,
- partitionNames.toSeq,
- uniqueId,
- partialMatch = true).collect()
- } catch {
- case e: Exception =>
- // roll back the drop partitions from carbon store
- new CarbonDropPartitionCommitRDD(sparkSession.sparkContext,
- table.getTablePath,
- segments.asScala,
- false,
- uniqueId,
- partitionNames.toSeq).collect()
- throw e
- }
- // commit the drop partitions from carbon store
- new CarbonDropPartitionCommitRDD(sparkSession.sparkContext,
+ // First drop the partitions from partition mapper files of each segment
+ val tuples = new CarbonDropPartitionRDD(sparkSession.sparkContext,
table.getTablePath,
segments.asScala,
- true,
- uniqueId,
- partitionNames.toSeq).collect()
- // Update the loadstatus with update time to clear cache from driver.
- val segmentSet = new util.HashSet[String](new SegmentStatusManager(table
- .getAbsoluteTableIdentifier).getValidAndInvalidSegments.getValidSegments)
- CarbonUpdateUtil.updateTableMetadataStatus(
- segmentSet,
- table,
- uniqueId,
- true,
- new util.ArrayList[String])
+ carbonPartitionsTobeDropped,
+ uniqueId).collect()
+ val tobeUpdatedSegs = new util.ArrayList[String]
+ val tobeDeletedSegs = new util.ArrayList[String]
+ tuples.foreach{case (tobeUpdated, tobeDeleted) =>
+ if (tobeUpdated.split(",").length > 0) {
+ tobeUpdatedSegs.add(tobeUpdated.split(",")(0))
+ }
+ if (tobeDeleted.split(",").length > 0) {
+ tobeDeletedSegs.add(tobeDeleted.split(",")(0))
+ }
+ }
+ SegmentFileStore.commitDropPartitions(table, uniqueId, tobeUpdatedSegs, tobeDeletedSegs)
DataMapStoreManager.getInstance().clearDataMaps(table.getAbsoluteTableIdentifier)
} finally {
AlterTableUtil.releaseLocks(locks)
- new PartitionMapFileStore().cleanSegments(
- table,
- new util.ArrayList(CarbonFilters.getPartitions(Seq.empty, sparkSession, tableName).asJava),
- false)
+ SegmentFileStore.cleanSegments(table, null, false)
}
Seq.empty[Row]
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
index 7fe2658..38ac58e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
@@ -31,7 +31,7 @@ import org.apache.spark.util.AlterTableUtil
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.cache.CacheProvider
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.datamap.{DataMapStoreManager, Segment}
import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
@@ -204,7 +204,7 @@ case class CarbonAlterTableDropPartitionCommand(
val validSegments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala
val threadArray: Array[Thread] = new Array[Thread](validSegments.size)
var i = 0
- for (segmentId: String <- validSegments) {
+ for (segmentId: Segment <- validSegments) {
threadArray(i) = dropPartitionThread(sqlContext, carbonLoadModel, executor,
segmentId, partitionId, dropWithData, oldPartitionIds)
threadArray(i).start()
@@ -216,7 +216,7 @@ case class CarbonAlterTableDropPartitionCommand(
val identifier = AbsoluteTableIdentifier.from(carbonLoadModel.getTablePath,
carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName)
val refresher = DataMapStoreManager.getInstance().getTableSegmentRefresher(identifier)
- refresher.refreshSegments(validSegments.asJava)
+ refresher.refreshSegments(validSegments.map(_.getSegmentNo).asJava)
} catch {
case e: Exception =>
LOGGER.error(s"Exception when dropping partition: ${ e.getMessage }")
@@ -238,7 +238,7 @@ case class CarbonAlterTableDropPartitionCommand(
case class dropPartitionThread(sqlContext: SQLContext,
carbonLoadModel: CarbonLoadModel,
executor: ExecutorService,
- segmentId: String,
+ segmentId: Segment,
partitionId: String,
dropWithData: Boolean,
oldPartitionIds: List[Int]) extends Thread {
@@ -259,7 +259,7 @@ case class dropPartitionThread(sqlContext: SQLContext,
private def executeDroppingPartition(sqlContext: SQLContext,
carbonLoadModel: CarbonLoadModel,
executor: ExecutorService,
- segmentId: String,
+ segmentId: Segment,
partitionId: String,
dropWithData: Boolean,
oldPartitionIds: List[Int]): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
index 020a72c..7aefbbe 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
@@ -215,7 +215,7 @@ case class CarbonAlterTableSplitPartitionCommand(
var i = 0
validSegments.foreach { segmentId =>
threadArray(i) = SplitThread(sqlContext, carbonLoadModel, executor,
- segmentId, partitionId, oldPartitionIdList)
+ segmentId.getSegmentNo, partitionId, oldPartitionIdList)
threadArray(i).start()
i += 1
}
@@ -225,7 +225,7 @@ case class CarbonAlterTableSplitPartitionCommand(
val identifier = AbsoluteTableIdentifier.from(carbonLoadModel.getTablePath,
carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName)
val refresher = DataMapStoreManager.getInstance().getTableSegmentRefresher(identifier)
- refresher.refreshSegments(validSegments.asJava)
+ refresher.refreshSegments(validSegments.map(_.getSegmentNo).asJava)
} catch {
case e: Exception =>
LOGGER.error(s"Exception when split partition: ${ e.getMessage }")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index d2acb00..59c43aa 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.execution.command.datamap.CarbonDropDataMapCommand
import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
import org.apache.spark.sql.execution.command.table.CarbonCreateTableCommand
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
+import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -178,7 +179,11 @@ case class CreatePreAggregateTableCommand(
// This will be used to check if the parent table has any segments or not. If not then no
// need to fire load for pre-aggregate table. Therefore reading the load details for PARENT
// table.
- DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false, parentTable)
+ DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false,
+ parentTable,
+ CarbonFilters.getCurrentPartitions(sparkSession,
+ TableIdentifier(parentTable.getTableName,
+ Some(parentTable.getDatabaseName))).map(_.asJava).orNull)
val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetaDataFilepath)
if (loadAvailable.exists(load => load.getSegmentStatus == SegmentStatus.INSERT_IN_PROGRESS ||
load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
index ed6be97..657e0c5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.execution.command.management.{CarbonAlterTableCompac
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable}
@@ -227,11 +228,17 @@ object LoadPostAggregateListener extends OperationEventListener {
operationContext.getProperty(
s"${parentTableDatabase}_${parentTableName}_Segment").toString)
} else {
- (TableIdentifier(table.getTableName, Some(table.getDatabaseName)),
- carbonLoadModel.getSegmentId)
+ val currentSegmentFile = operationContext.getProperty("current.segmentfile")
+ val segment = if (currentSegmentFile != null) {
+ new Segment(carbonLoadModel.getSegmentId, currentSegmentFile.toString)
+ } else {
+ Segment.toSegment(carbonLoadModel.getSegmentId)
+ }
+ (TableIdentifier(table.getTableName, Some(table.getDatabaseName)), segment.toString)
}
+
PreAggregateUtil.startDataLoadForDataMap(
- parentTableIdentifier,
+ parentTableIdentifier,
segmentToLoad,
validateSegments = false,
childLoadCommand,
[10/16] carbondata git commit: [CARBONDATA-2187][PARTITION] Partition
restructure for new folder structure and supporting partition location
feature
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 4caa401..35a3513 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
import org.apache.spark.sql.SparkSession
import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.block.SegmentProperties
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
@@ -197,8 +198,8 @@ class StreamHandoffRDD[K, V](
override protected def getPartitions: Array[Partition] = {
val job = Job.getInstance(FileFactory.getConfiguration)
val inputFormat = new CarbonTableInputFormat[Array[Object]]()
- val segmentList = new util.ArrayList[String](1)
- segmentList.add(handOffSegmentId)
+ val segmentList = new util.ArrayList[Segment](1)
+ segmentList.add(Segment.toSegment(handOffSegmentId))
val splits = inputFormat.getSplitsOfStreaming(
job,
carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getAbsoluteTableIdentifier,
[06/16] carbondata git commit: [CARBONDATA-2201] NPE fixed while
triggering the LoadTablePreExecutionEvent before Streaming
Posted by gv...@apache.org.
[CARBONDATA-2201] NPE fixed while triggering the LoadTablePreExecutionEvent before Streaming
While triggering the LoadTablePreExecutionEvent we require options provided by user and the finalOptions.
In case of streaming both are same. If we pass null . It may cause NPE.
This closes #1997
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e363dd1a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e363dd1a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e363dd1a
Branch: refs/heads/branch-1.3
Commit: e363dd1a68e2138591a930055dd1701a1245825f
Parents: b0a2fab
Author: rahulforallp <ra...@knoldus.in>
Authored: Sun Feb 25 15:25:26 2018 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Tue Feb 27 12:51:52 2018 +0530
----------------------------------------------------------------------
.../scala/org/apache/carbondata/streaming/StreamSinkFactory.scala | 3 ++-
.../sql/execution/streaming/CarbonAppendableStreamSink.scala | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e363dd1a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index 1c7be6a..3366f51 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -60,6 +60,7 @@ object StreamSinkFactory {
parameters,
"")
// fire pre event before streamin is started
+ // in case of streaming options and optionsFinal can be same
val operationContext = new OperationContext
val loadTablePreExecutionEvent = new LoadTablePreExecutionEvent(
carbonTable.getCarbonTableIdentifier,
@@ -67,7 +68,7 @@ object StreamSinkFactory {
carbonLoadModel.getFactFilePath,
false,
parameters.asJava,
- null,
+ parameters.asJava,
false
)
OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e363dd1a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index 849bf99..67d8a4d 100644
--- a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++ b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -95,6 +95,7 @@ class CarbonAppendableStreamSink(
val statistic = new QueryStatistic()
// fire pre event on every batch add
+ // in case of streaming options and optionsFinal can be same
val operationContext = new OperationContext
val loadTablePreExecutionEvent = new LoadTablePreExecutionEvent(
carbonTable.getCarbonTableIdentifier,
@@ -102,7 +103,7 @@ class CarbonAppendableStreamSink(
carbonLoadModel.getFactFilePath,
false,
parameters.asJava,
- null,
+ parameters.asJava,
false
)
OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext)
[05/16] carbondata git commit: [CARBONDATA-2200] Fix bug of LIKE
operation on streaming table
Posted by gv...@apache.org.
[CARBONDATA-2200] Fix bug of LIKE operation on streaming table
Fix bug of LIKE operation on streaming table,
LIKE operation will be converted to StartsWith / EndsWith / Contains expression.
Carbon will use RowLevelFilterExecuterImpl to evaluate this expression.
Streaming table also should implement RowLevelFilterExecuterImpl.
This closes #1996
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b0a2fabc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b0a2fabc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b0a2fabc
Branch: refs/heads/branch-1.3
Commit: b0a2fabcc8584dfba24ad0ea135948f5365a7335
Parents: 6f9016d
Author: QiangCai <qi...@qq.com>
Authored: Sun Feb 25 18:53:41 2018 +0800
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Tue Feb 27 12:51:38 2018 +0530
----------------------------------------------------------------------
.../executer/RowLevelFilterExecuterImpl.java | 82 +++++++++++++++++++-
.../TestStreamingTableOperation.scala | 20 ++++-
2 files changed, 100 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0a2fabc/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 8956f30..de97e82 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -60,6 +60,7 @@ import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnRes
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.core.util.BitSetGroup;
+import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -276,13 +277,92 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
try {
- return exp.evaluate(value).getBoolean();
+ Boolean result = exp.evaluate(convertRow(value, dimOrdinalMax)).getBoolean();
+ return result == null ? false : result;
} catch (FilterIllegalMemberException e) {
throw new FilterUnsupportedException(e);
}
}
/**
+ * convert encoded row to actual value row for filter to evaluate expression
+ * @param value this row will be converted to actual value
+ * @param dimOrdinalMax for measure column, its index in row = dimOrdinalMax + its ordinal
+ * @return actual value row
+ * @throws IOException
+ */
+ private RowIntf convertRow(RowIntf value, int dimOrdinalMax) throws IOException {
+ Object[] record = new Object[value.size()];
+ String memberString;
+ for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
+ int index = dimColumnEvaluatorInfo.getDimension().getOrdinal();
+ // if filter dimension is not present in the current add its default value
+ if (!isDimensionPresentInCurrentBlock[i]) {
+ // fill default value here
+ record[index] = getDimensionDefaultValue(dimColumnEvaluatorInfo);
+ // already set value, so continue to set next dimension
+ continue;
+ }
+ if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) {
+ if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
+ record[index] = dimColumnEvaluatorInfo.getDimension().getDefaultValue();
+ }
+ byte[] memberBytes = (byte[]) value.getVal(index);
+ if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ // no dictionary
+ if (null != memberBytes) {
+ if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
+ memberBytes = null;
+ } else if (memberBytes.length == 0) {
+ memberBytes = null;
+ }
+ record[index] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes,
+ dimColumnEvaluatorInfo.getDimension().getDataType());
+ }
+ } else {
+ // dictionary
+ int dictionaryValue = ByteUtil.toInt(memberBytes, 0);
+ if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
+ && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ memberString =
+ getFilterActualValueFromDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
+ record[index] = DataTypeUtil.getDataBasedOnDataType(memberString,
+ dimColumnEvaluatorInfo.getDimension().getDataType());
+ } else if (
+ dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ Object member = getFilterActualValueFromDirectDictionaryValue(dimColumnEvaluatorInfo,
+ dictionaryValue);
+ record[index] = member;
+ }
+ }
+ } else {
+ // complex
+ record[index] = value.getVal(index);
+ }
+ }
+
+ for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
+ MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
+ int index = msrColumnEvalutorInfo.getMeasure().getOrdinal() + dimOrdinalMax;
+ // add default value for the measure in case filter measure is not present
+ // in the current block measure list
+ if (!isMeasurePresentInCurrentBlock[i]) {
+ byte[] defaultValue = msrColumnEvalutorInfo.getCarbonColumn().getDefaultValue();
+ record[index] = RestructureUtil.getMeasureDefaultValue(
+ msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema(), defaultValue);
+ // already set value, so continue to set next measure
+ continue;
+ }
+ // measure
+ record[index] = value.getVal(index);
+ }
+ RowIntf row = new RowImpl();
+ row.setValues(record);
+ return row;
+ }
+
+ /**
* Method will read the members of particular dimension block and create
* a row instance for further processing of the filters
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b0a2fabc/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 94baf86..5644302 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -332,6 +332,12 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
sql("select * from stream_table_filter where name like '%me_3%' and id < 30"),
Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+ checkAnswer(sql("select count(*) from stream_table_filter where name like '%ame%'"),
+ Seq(Row(49)))
+
+ checkAnswer(sql("select count(*) from stream_table_filter where name like '%batch%'"),
+ Seq(Row(5)))
+
checkAnswer(
sql("select * from stream_table_filter where name >= 'name_3' and id < 4"),
Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
@@ -350,6 +356,9 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0")),
Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+ checkAnswer(sql("select count(*) from stream_table_filter where city like '%city%'"),
+ Seq(Row(54)))
+
checkAnswer(
sql("select * from stream_table_filter where city > 'city_09' and city < 'city_10'"),
Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0")),
@@ -649,6 +658,12 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
sql("select * from stream_table_filter_complex where name like '%me_3%' and id < 30"),
Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_3", "school_33")), 3))))
+ checkAnswer(sql("select count(*) from stream_table_filter_complex where name like '%ame%'"),
+ Seq(Row(49)))
+
+ checkAnswer(sql("select count(*) from stream_table_filter_complex where name like '%batch%'"),
+ Seq(Row(5)))
+
checkAnswer(
sql("select * from stream_table_filter_complex where name >= 'name_3' and id < 4"),
Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_3", "school_33")), 3))))
@@ -663,6 +678,9 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 1)),
Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 20))))
+ checkAnswer(sql("select count(*) from stream_table_filter_complex where city like '%city%'"),
+ Seq(Row(54)))
+
checkAnswer(
sql("select * from stream_table_filter_complex where city > 'city_09' and city < 'city_10'"),
Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 1)),
@@ -1056,7 +1074,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
//Verify MergeTO column entry for compacted Segments
newSegments.filter(_.getString(1).equals("Compacted")).foreach{ rw =>
assertResult("Compacted")(rw.getString(1))
- assertResult((Integer.parseInt(rw.getString(0))+2).toString)(rw.getString(4))
+ assert(Integer.parseInt(rw.getString(0)) < Integer.parseInt(rw.getString(4)))
}
checkAnswer(
sql("select count(*) from streaming.stream_table_reopen"),
[11/16] carbondata git commit: [CARBONDATA-2187][PARTITION] Partition
restructure for new folder structure and supporting partition location
feature
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index fc7c13a..22dab27 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -28,6 +28,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.exception.InvalidConfigurationException
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.util.CarbonUtil
@@ -98,6 +99,18 @@ case class CarbonCreateTableCommand(
val partitionString =
if (partitionInfo != null &&
partitionInfo.getPartitionType == PartitionType.NATIVE_HIVE) {
+ // Restrict dictionary encoding on partition columns.
+ // TODO Need to decide wherher it is required
+ val dictionaryOnPartitionColumn =
+ partitionInfo.getColumnSchemaList.asScala.exists{p =>
+ p.hasEncoding(Encoding.DICTIONARY) && !p.hasEncoding(Encoding.DIRECT_DICTIONARY)
+ }
+ if (dictionaryOnPartitionColumn) {
+ throwMetadataException(
+ dbName,
+ tableName,
+ s"Dictionary include cannot be applied on partition columns")
+ }
s" PARTITIONED BY (${partitionInfo.getColumnSchemaList.asScala.map(
_.getColumnName).mkString(",")})"
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index d2c691b..bff65be 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -17,7 +17,6 @@
package org.apache.spark.sql.execution.datasources
import java.io.File
-import java.text.SimpleDateFormat
import java.util
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicLong
@@ -39,16 +38,18 @@ import org.apache.spark.sql.sources.DataSourceRegister
import org.apache.spark.sql.types._
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
-import org.apache.carbondata.core.metadata.PartitionMapFileStore
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.indexstore
+import org.apache.carbondata.core.metadata.SegmentFileStore
import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.util.{CarbonProperties, DataTypeUtil}
+import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager}
+import org.apache.carbondata.core.util.{CarbonProperties, DataTypeConverterImpl, DataTypeUtil}
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.hadoop.api.{CarbonOutputCommitter, CarbonTableOutputFormat}
import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat.CarbonRecordWriter
import org.apache.carbondata.hadoop.internal.ObjectArrayWritable
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil
-import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.spark.util.{CarbonScalaUtil, DataLoadingUtil, Util}
@@ -66,6 +67,10 @@ with Serializable {
None
}
+ SparkSession.getActiveSession.get.sessionState.conf.setConfString(
+ "spark.sql.sources.commitProtocolClass",
+ "org.apache.spark.sql.execution.datasources.CarbonSQLHadoopMapReduceCommitProtocol")
+
override def prepareWrite(
sparkSession: SparkSession,
job: Job,
@@ -77,9 +82,6 @@ with Serializable {
classOf[CarbonOutputCommitter],
classOf[CarbonOutputCommitter])
conf.set("carbon.commit.protocol", "carbon.commit.protocol")
- sparkSession.sessionState.conf.setConfString(
- "spark.sql.sources.commitProtocolClass",
- "org.apache.spark.sql.execution.datasources.CarbonSQLHadoopMapReduceCommitProtocol")
job.setOutputFormatClass(classOf[CarbonTableOutputFormat])
val table = CarbonEnv.getCarbonTable(
TableIdentifier(options("tableName"), options.get("dbName")))(sparkSession)
@@ -114,13 +116,7 @@ with Serializable {
model.setDictionaryServerPort(options.getOrElse("dictport", "-1").toInt)
CarbonTableOutputFormat.setOverwrite(conf, options("overwrite").toBoolean)
model.setPartitionLoad(true)
- // Set the update timestamp if user sets in case of update query. It needs to be updated
- // in load status update time
- val updateTimeStamp = options.get("updatetimestamp")
- if (updateTimeStamp.isDefined) {
- conf.set(CarbonTableOutputFormat.UPADTE_TIMESTAMP, updateTimeStamp.get)
- model.setFactTimeStamp(updateTimeStamp.get.toLong)
- }
+
val staticPartition = options.getOrElse("staticpartition", null)
if (staticPartition != null) {
conf.set("carbon.staticpartition", staticPartition)
@@ -131,6 +127,30 @@ with Serializable {
if (segemntsTobeDeleted.isDefined) {
conf.set(CarbonTableOutputFormat.SEGMENTS_TO_BE_DELETED, segemntsTobeDeleted.get)
}
+
+ val currPartition = options.getOrElse("currentpartition", null)
+ if (currPartition != null) {
+ conf.set("carbon.currentpartition", currPartition)
+ }
+ // Update with the current in progress load.
+ val currEntry = options.getOrElse("currentloadentry", null)
+ if (currEntry != null) {
+ val loadEntry =
+ ObjectSerializationUtil.convertStringToObject(currEntry).asInstanceOf[LoadMetadataDetails]
+ val details =
+ SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(model.getTablePath))
+ model.setSegmentId(loadEntry.getLoadName)
+ model.setFactTimeStamp(loadEntry.getLoadStartTime)
+ val list = new util.ArrayList[LoadMetadataDetails](details.toList.asJava)
+ list.add(loadEntry)
+ model.setLoadMetadataDetails(list)
+ }
+ // Set the update timestamp if user sets in case of update query. It needs to be updated
+ // in load status update time
+ val updateTimeStamp = options.get("updatetimestamp")
+ if (updateTimeStamp.isDefined) {
+ conf.set(CarbonTableOutputFormat.UPADTE_TIMESTAMP, updateTimeStamp.get)
+ }
CarbonTableOutputFormat.setLoadModel(conf, model)
new OutputWriterFactory {
@@ -146,13 +166,14 @@ with Serializable {
path: String,
dataSchema: StructType,
context: TaskAttemptContext): OutputWriter = {
+ val model = CarbonTableOutputFormat.getLoadModel(context.getConfiguration)
val isCarbonUseMultiDir = CarbonProperties.getInstance().isUseMultiTempDir
var storeLocation: Array[String] = Array[String]()
val isCarbonUseLocalDir = CarbonProperties.getInstance()
.getProperty("carbon.use.local.dir", "false").equalsIgnoreCase("true")
- val taskNumber = generateTaskNumber(path, context)
+ val taskNumber = generateTaskNumber(path, context, model.getSegmentId)
val tmpLocationSuffix =
File.separator + "carbon" + System.nanoTime() + File.separator + taskNumber
if (isCarbonUseLocalDir) {
@@ -174,7 +195,7 @@ with Serializable {
storeLocation :+ (System.getProperty("java.io.tmpdir") + tmpLocationSuffix)
}
CarbonTableOutputFormat.setTempStoreLocations(context.getConfiguration, storeLocation)
- new CarbonOutputWriter(path, context, dataSchema.map(_.dataType), taskNumber)
+ new CarbonOutputWriter(path, context, dataSchema.map(_.dataType), taskNumber, model)
}
/**
@@ -182,7 +203,7 @@ with Serializable {
* of partition tables.
*/
private def generateTaskNumber(path: String,
- context: TaskAttemptContext): String = {
+ context: TaskAttemptContext, segmentId: String): String = {
var partitionNumber: java.lang.Long = taskIdMap.get(path)
if (partitionNumber == null) {
partitionNumber = counter.incrementAndGet()
@@ -190,8 +211,7 @@ with Serializable {
taskIdMap.put(path, partitionNumber)
}
val taskID = context.getTaskAttemptID.getTaskID.getId
- String.valueOf(Math.pow(10, 5).toInt + taskID) +
- String.valueOf(partitionNumber + Math.pow(10, 5).toInt)
+ CarbonScalaUtil.generateUniqueNumber(taskID, segmentId, partitionNumber)
}
override def getFileExtension(context: TaskAttemptContext): String = {
@@ -233,9 +253,12 @@ private trait AbstractCarbonOutputWriter {
private class CarbonOutputWriter(path: String,
context: TaskAttemptContext,
fieldTypes: Seq[DataType],
- taskNo : String)
+ taskNo : String,
+ model: CarbonLoadModel)
extends OutputWriter with AbstractCarbonOutputWriter {
- val model = CarbonTableOutputFormat.getLoadModel(context.getConfiguration)
+
+ val converter = new DataTypeConverterImpl
+
val partitions =
getPartitionsFromPath(path, context, model).map(ExternalCatalogUtils.unescapePathName)
val staticPartition: util.HashMap[String, Boolean] = {
@@ -247,47 +270,52 @@ private class CarbonOutputWriter(path: String,
null
}
}
- lazy val (updatedPartitions, partitionData) = if (partitions.nonEmpty) {
- val updatedPartitions = partitions.map{ p =>
- val value = p.substring(p.indexOf("=") + 1, p.length)
- val col = p.substring(0, p.indexOf("="))
- // NUll handling case. For null hive creates with this special name
- if (value.equals("__HIVE_DEFAULT_PARTITION__")) {
- (col, null)
- // we should replace back the special string with empty value.
- } else if (value.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- (col, "")
- } else {
- (col, value)
- }
- }
-
- if (staticPartition != null) {
- val loadModel = recordWriter.getLoadModel
- val table = loadModel.getCarbonDataLoadSchema.getCarbonTable
- var timeStampformatString = loadModel.getTimestampformat
- if (timeStampformatString.isEmpty) {
- timeStampformatString = loadModel.getDefaultTimestampFormat
- }
- val timeFormat = new SimpleDateFormat(timeStampformatString)
- var dateFormatString = loadModel.getDateFormat
- if (dateFormatString.isEmpty) {
- dateFormatString = loadModel.getDefaultDateFormat
- }
- val dateFormat = new SimpleDateFormat(dateFormatString)
- val formattedPartitions = updatedPartitions.map {case (col, value) =>
- // Only convert the static partitions to the carbon format and use it while loading data
- // to carbon.
- (col, value)
- }
- (formattedPartitions, updatePartitions(formattedPartitions.map(_._2)))
+ lazy val currPartitions: util.List[indexstore.PartitionSpec] = {
+ val currParts = context.getConfiguration.get("carbon.currentpartition")
+ if (currParts != null) {
+ ObjectSerializationUtil.convertStringToObject(
+ currParts).asInstanceOf[util.List[indexstore.PartitionSpec]]
} else {
- (updatedPartitions, updatePartitions(updatedPartitions.map(_._2)))
+ new util.ArrayList[indexstore.PartitionSpec]()
}
+ }
+ var (updatedPartitions, partitionData) = if (partitions.nonEmpty) {
+ val updatedPartitions = partitions.map(splitPartition)
+ (updatedPartitions, updatePartitions(updatedPartitions.map(_._2)))
} else {
(Map.empty[String, String].toArray, Array.empty)
}
+ private def splitPartition(p: String) = {
+ val value = p.substring(p.indexOf("=") + 1, p.length)
+ val col = p.substring(0, p.indexOf("="))
+ // NUll handling case. For null hive creates with this special name
+ if (value.equals("__HIVE_DEFAULT_PARTITION__")) {
+ (col, null)
+ // we should replace back the special string with empty value.
+ } else if (value.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ (col, "")
+ } else {
+ (col, value)
+ }
+ }
+
+ lazy val writePath = {
+ val updatedPath = getPartitionPath(path, context, model)
+ // in case of partition location specified by user then search the partitions from the current
+ // partitions to get the corresponding partitions.
+ if (partitions.isEmpty) {
+ val writeSpec = new indexstore.PartitionSpec(null, updatedPath)
+ val index = currPartitions.indexOf(writeSpec)
+ if (index > -1) {
+ val spec = currPartitions.get(index)
+ updatedPartitions = spec.getPartitions.asScala.map(splitPartition).toArray
+ partitionData = updatePartitions(updatedPartitions.map(_._2))
+ }
+ }
+ updatedPath
+ }
+
val writable = new ObjectArrayWritable
private def updatePartitions(partitionData: Seq[String]): Array[AnyRef] = {
@@ -302,21 +330,30 @@ private class CarbonOutputWriter(path: String,
} else {
col.getDataType
}
- if (staticPartition != null) {
- DataTypeUtil.getDataBasedOnDataType(
+ if (staticPartition != null && staticPartition.get(col.getColumnName.toLowerCase)) {
+ val converetedVal =
CarbonScalaUtil.convertStaticPartitions(
partitionData(index),
col,
- model.getCarbonDataLoadSchema.getCarbonTable),
- dataType)
+ model.getCarbonDataLoadSchema.getCarbonTable)
+ if (col.hasEncoding(Encoding.DICTIONARY)) {
+ converetedVal.toInt.asInstanceOf[AnyRef]
+ } else {
+ DataTypeUtil.getDataBasedOnDataType(
+ converetedVal,
+ dataType,
+ converter)
+ }
} else {
- DataTypeUtil.getDataBasedOnDataType(partitionData(index), dataType)
+ DataTypeUtil.getDataBasedOnDataType(partitionData(index), dataType, converter)
}
}.toArray
}
private val recordWriter: CarbonRecordWriter = {
context.getConfiguration.set("carbon.outputformat.taskno", taskNo)
+ context.getConfiguration.set("carbon.outputformat.writepath",
+ writePath + "/" + model.getSegmentId + "_" + model.getFactTimeStamp + ".tmp")
new CarbonTableOutputFormat() {
override def getDefaultWorkFile(context: TaskAttemptContext, extension: String): Path = {
new Path(path)
@@ -355,51 +392,54 @@ private class CarbonOutputWriter(path: String,
override def close(): Unit = {
recordWriter.close(context)
- val loadModel = recordWriter.getLoadModel
- val segmentPath = CarbonTablePath.getSegmentPath(loadModel.getTablePath, loadModel.getSegmentId)
- val table = loadModel.getCarbonDataLoadSchema.getCarbonTable
- var timeStampformatString = loadModel.getTimestampformat
- if (timeStampformatString.isEmpty) {
- timeStampformatString = loadModel.getDefaultTimestampFormat
- }
- val timeFormat = new SimpleDateFormat(timeStampformatString)
- var dateFormatString = loadModel.getDateFormat
- if (dateFormatString.isEmpty) {
- dateFormatString = loadModel.getDefaultDateFormat
- }
- val dateFormat = new SimpleDateFormat(dateFormatString)
- val serializeFormat =
- loadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
- val badRecordAction = loadModel.getBadRecordsAction.split(",")(1)
- val isEmptyBadRecord = loadModel.getIsEmptyDataBadRecord.split(",")(1).toBoolean
// write partition info to new file.
val partitonList = new util.ArrayList[String]()
val formattedPartitions =
// All dynamic partitions need to be converted to proper format
CarbonScalaUtil.updatePartitions(
updatedPartitions.toMap,
- table,
- timeFormat,
- dateFormat)
+ model.getCarbonDataLoadSchema.getCarbonTable)
formattedPartitions.foreach(p => partitonList.add(p._1 + "=" + p._2))
- new PartitionMapFileStore().writePartitionMapFile(
- segmentPath,
- loadModel.getTaskNo,
+ SegmentFileStore.writeSegmentFile(
+ model.getTablePath,
+ taskNo,
+ writePath,
+ model.getSegmentId + "_" + model.getFactTimeStamp + "",
partitonList)
}
+ def getPartitionPath(path: String,
+ attemptContext: TaskAttemptContext,
+ model: CarbonLoadModel): String = {
+ if (updatedPartitions.nonEmpty) {
+ val formattedPartitions =
+ // All dynamic partitions need to be converted to proper format
+ CarbonScalaUtil.updatePartitions(
+ updatedPartitions.toMap,
+ model.getCarbonDataLoadSchema.getCarbonTable)
+ val partitionstr = formattedPartitions.map{p =>
+ ExternalCatalogUtils.escapePathName(p._1) + "=" + ExternalCatalogUtils.escapePathName(p._2)
+ }.mkString(CarbonCommonConstants.FILE_SEPARATOR)
+ model.getCarbonDataLoadSchema.getCarbonTable.getTablePath +
+ CarbonCommonConstants.FILE_SEPARATOR + partitionstr
+ } else {
+ var updatedPath = FileFactory.getUpdatedFilePath(path)
+ updatedPath.substring(0, updatedPath.lastIndexOf("/"))
+ }
+ }
+
def getPartitionsFromPath(
path: String,
attemptContext: TaskAttemptContext,
model: CarbonLoadModel): Array[String] = {
var attemptId = attemptContext.getTaskAttemptID.toString + "/"
- if (path.indexOf(attemptId) <= 0) {
-
- attemptId = model.getTableName + "/"
- }
- val str = path.substring(path.indexOf(attemptId) + attemptId.length, path.lastIndexOf("/"))
- if (str.length > 0) {
- str.split("/")
+ if (path.indexOf(attemptId) > -1) {
+ val str = path.substring(path.indexOf(attemptId) + attemptId.length, path.lastIndexOf("/"))
+ if (str.length > 0) {
+ str.split("/")
+ } else {
+ Array.empty
+ }
} else {
Array.empty
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index 544c494..48679b1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.schema.BucketingInfo
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager
@@ -143,10 +144,10 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
projects: Seq[NamedExpression],
filterPredicates: Seq[Expression],
scanBuilder: (Seq[Attribute], Array[Filter],
- ArrayBuffer[AttributeReference], Seq[String]) => RDD[InternalRow]) = {
+ ArrayBuffer[AttributeReference], Seq[PartitionSpec]) => RDD[InternalRow]) = {
val names = relation.catalogTable.get.partitionColumnNames
// Get the current partitions from table.
- var partitions: Seq[String] = null
+ var partitions: Seq[PartitionSpec] = null
if (names.nonEmpty) {
val partitionSet = AttributeSet(names
.map(p => relation.output.find(_.name.equalsIgnoreCase(p)).get))
@@ -167,7 +168,7 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
CarbonFilters.getPartitions(
updatedPartitionFilters.toSeq,
SparkSession.getActiveSession.get,
- relation.catalogTable.get.identifier)
+ relation.catalogTable.get.identifier).orNull
}
pruneFilterProjectRaw(
relation,
@@ -199,9 +200,9 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
relation: LogicalRelation,
rawProjects: Seq[NamedExpression],
filterPredicates: Seq[Expression],
- partitions: Seq[String],
+ partitions: Seq[PartitionSpec],
scanBuilder: (Seq[Attribute], Seq[Expression], Seq[Filter],
- ArrayBuffer[AttributeReference], Seq[String]) => RDD[InternalRow]) = {
+ ArrayBuffer[AttributeReference], Seq[PartitionSpec]) => RDD[InternalRow]) = {
val projects = rawProjects.map {p =>
p.transform {
case CustomDeterministicExpression(exp) => exp
@@ -350,9 +351,9 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
private def getDataSourceScan(relation: LogicalRelation,
output: Seq[Attribute],
- partitions: Seq[String],
+ partitions: Seq[PartitionSpec],
scanBuilder: (Seq[Attribute], Seq[Expression], Seq[Filter],
- ArrayBuffer[AttributeReference], Seq[String]) => RDD[InternalRow],
+ ArrayBuffer[AttributeReference], Seq[PartitionSpec]) => RDD[InternalRow],
candidatePredicates: Seq[Expression],
pushedFilters: Seq[Filter],
metadata: Map[String, String],
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 0178716..f69ccc1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy}
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.command.management.{CarbonAlterTableCompactionCommand, CarbonInsertIntoCommand, CarbonLoadDataCommand, RefreshCarbonTableCommand}
-import org.apache.spark.sql.execution.command.partition.{CarbonAlterTableDropHivePartitionCommand, CarbonShowCarbonPartitionsCommand}
+import org.apache.spark.sql.execution.command.partition.{CarbonAlterTableAddHivePartitionCommand, CarbonAlterTableDropHivePartitionCommand, CarbonShowCarbonPartitionsCommand}
import org.apache.spark.sql.execution.command.schema._
import org.apache.spark.sql.execution.command.table.{CarbonDescribeFormattedCommand, CarbonDropTableCommand}
import org.apache.spark.sql.hive.execution.command.{CarbonDropDatabaseCommand, CarbonResetCommand, CarbonSetCommand}
@@ -247,16 +247,19 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
} else {
ExecutedCommandExec(rename) :: Nil
}
- case addPartition@AlterTableAddPartitionCommand(tableName, partitionSpecsAndLocs, _) =>
+ case addPart@AlterTableAddPartitionCommand(tableName, partitionSpecsAndLocs, ifNotExists) =>
val dbOption = tableName.database.map(_.toLowerCase)
val tableIdentifier = TableIdentifier(tableName.table.toLowerCase(), dbOption)
val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
.tableExists(tableIdentifier)(sparkSession)
- if (isCarbonTable && partitionSpecsAndLocs.exists(_._2.isDefined)) {
- throw new UnsupportedOperationException(
- "add partition with location is not supported")
+ if (isCarbonTable) {
+ ExecutedCommandExec(
+ CarbonAlterTableAddHivePartitionCommand(
+ tableName,
+ partitionSpecsAndLocs,
+ ifNotExists)) :: Nil
} else {
- ExecutedCommandExec(addPartition) :: Nil
+ ExecutedCommandExec(addPart) :: Nil
}
case RefreshTable(tableIdentifier) =>
RefreshCarbonTableCommand(tableIdentifier.database,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
index b8608f4..7bf8536 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
@@ -29,11 +29,14 @@ import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Stati
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.CarbonException
+import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.SegmentFileStore
import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
+import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
/**
@@ -209,9 +212,10 @@ case class CarbonRelation(
.getValidAndInvalidSegments.getValidSegments.isEmpty) {
sizeInBytesLocalValue = 0L
} else {
- val tablePath = CarbonStorePath.getCarbonTablePath(
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(
carbonTable.getTablePath,
- carbonTable.getCarbonTableIdentifier).getPath
+ carbonTable.getCarbonTableIdentifier)
+ val tablePath = carbonTablePath.getPath
val fileType = FileFactory.getFileType(tablePath)
if (FileFactory.isFileExist(tablePath, fileType)) {
// get the valid segments
@@ -220,8 +224,14 @@ case class CarbonRelation(
var size = 0L
// for each segment calculate the size
segments.foreach {validSeg =>
- size = size + FileFactory.getDirectorySize(
- CarbonTablePath.getSegmentPath(tablePath, validSeg))
+ if (validSeg.getSegmentFileName != null) {
+ val fileStore = new SegmentFileStore(tablePath, validSeg.getSegmentFileName)
+ size = size + CarbonUtil.getSizeOfSegment(
+ carbonTablePath, new Segment(validSeg.getSegmentNo, validSeg.getSegmentFileName))
+ } else {
+ size = size + FileFactory.getDirectorySize(
+ CarbonTablePath.getSegmentPath(tablePath, validSeg.getSegmentNo))
+ }
}
// update the new table status time
tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index c7767ce..e5eb53c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.optimizer
+import java.util
+
import scala.collection.JavaConverters._
import org.apache.spark.sql._
@@ -30,14 +32,15 @@ import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.hive.CarbonSessionCatalog
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.PartitionMapFileStore
+import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.indexstore.PartitionSpec
+import org.apache.carbondata.core.metadata.SegmentFileStore
import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
import org.apache.carbondata.core.scan.expression.conditional._
import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
-import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.util.CarbonScalaUtil
@@ -432,15 +435,11 @@ object CarbonFilters {
}
def getCurrentPartitions(sparkSession: SparkSession,
- carbonTable: CarbonTable): Seq[String] = {
- if (carbonTable.isHivePartitionTable) {
- CarbonFilters.getPartitions(
- Seq.empty,
- sparkSession,
- TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName)))
- } else {
- Seq.empty
- }
+ tableIdentifier: TableIdentifier): Option[Seq[PartitionSpec]] = {
+ CarbonFilters.getPartitions(
+ Seq.empty,
+ sparkSession,
+ tableIdentifier)
}
/**
@@ -452,11 +451,15 @@ object CarbonFilters {
*/
def getPartitions(partitionFilters: Seq[Expression],
sparkSession: SparkSession,
- identifier: TableIdentifier): Seq[String] = {
+ identifier: TableIdentifier): Option[Seq[PartitionSpec]] = {
+ val table = CarbonEnv.getCarbonTable(identifier)(sparkSession)
// first try to read partitions in case if the trigger comes from the aggregation table load.
- val partitionsForAggTable = getPartitionsForAggTable(sparkSession, identifier)
+ val partitionsForAggTable = getPartitionsForAggTable(sparkSession, table)
if (partitionsForAggTable.isDefined) {
- return partitionsForAggTable.get
+ return partitionsForAggTable
+ }
+ if (!table.isHivePartitionTable) {
+ return None
}
val partitions = {
try {
@@ -483,35 +486,33 @@ object CarbonFilters {
identifier)
}
}
- partitions.toList.flatMap { partition =>
- partition.spec.seq.map{case (column, value) => column + "=" + value}
- }.toSet.toSeq
+ Some(partitions.map { partition =>
+ new PartitionSpec(
+ new util.ArrayList[String]( partition.spec.seq.map{case (column, value) =>
+ column + "=" + value}.toList.asJava), partition.location)
+ })
}
/**
* In case of loading aggregate tables it needs to be get only from the main table load in
- * progress segment. So we should read from the partition map file of that segment.
+ * progress segment. So we should read from the segment file of that segment
*/
def getPartitionsForAggTable(sparkSession: SparkSession,
- identifier: TableIdentifier): Option[Seq[String]] = {
+ table: CarbonTable): Option[Seq[PartitionSpec]] = {
// when validate segments is disabled then only read from partitionmap
val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
if (carbonSessionInfo != null) {
val validateSegments = carbonSessionInfo.getSessionParams
.getProperty(CarbonCommonConstants.VALIDATE_CARBON_INPUT_SEGMENTS +
- CarbonEnv.getDatabaseName(identifier.database)(sparkSession) + "." +
- identifier.table, "true").toBoolean
+ table.getDatabaseName + "." +
+ table.getTableName, "true").toBoolean
if (!validateSegments) {
val segmentNumbersFromProperty = CarbonProperties.getInstance
.getProperty(CarbonCommonConstants.CARBON_INPUT_SEGMENTS +
- CarbonEnv.getDatabaseName(identifier.database)(sparkSession)
- + "." + identifier.table)
- val carbonTable = CarbonEnv.getCarbonTable(identifier)(sparkSession)
- val segmentPath =
- CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentNumbersFromProperty)
- val partitionMapper = new PartitionMapFileStore()
- partitionMapper.readAllPartitionsOfSegment(segmentPath)
- Some(partitionMapper.getPartitionMap.asScala.map(_._2).flatMap(_.asScala).toSet.toSeq)
+ table.getDatabaseName + "." + table.getTableName)
+ val segment = Segment.toSegment(segmentNumbersFromProperty)
+ val segmentFile = new SegmentFileStore(table.getTablePath, segment.getSegmentFileName)
+ Some(segmentFile.getPartitionSpecs.asScala)
} else {
None
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
index 0b62e10..d45020b 100644
--- a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -38,7 +38,7 @@ import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, Experime
import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.util.CarbonScalaUtil
/**
@@ -146,14 +146,8 @@ class CarbonSessionCatalog(
ignoreIfExists: Boolean): Unit = {
try {
val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
- // Get the properties from thread local
- val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
- if (carbonSessionInfo != null) {
- val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
- super.createPartitions(tableName, updatedParts, ignoreIfExists)
- } else {
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
+ val updatedParts = CarbonScalaUtil.updatePartitions(parts, table)
+ super.createPartitions(tableName, updatedParts, ignoreIfExists)
} catch {
case e: Exception =>
super.createPartitions(tableName, parts, ignoreIfExists)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
index baadd04..b9425d6 100644
--- a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -46,7 +46,7 @@ import org.apache.spark.util.CarbonReflectionUtils
import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CarbonScalaUtil
@@ -144,14 +144,8 @@ class CarbonSessionCatalog(
ignoreIfExists: Boolean): Unit = {
try {
val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
- // Get the properties from thread local
- val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
- if (carbonSessionInfo != null) {
- val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
- super.createPartitions(tableName, updatedParts, ignoreIfExists)
- } else {
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
+ val updatedParts = CarbonScalaUtil.updatePartitions(parts, table)
+ super.createPartitions(tableName, updatedParts, ignoreIfExists)
} catch {
case e: Exception =>
super.createPartitions(tableName, parts, ignoreIfExists)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 4b0113c..d739f8c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datamap.dev.DataMapWriter;
@@ -71,7 +72,7 @@ public class DataMapWriterListener {
}
List<String> columns = factory.getMeta().getIndexedColumns();
List<DataMapWriter> writers = registry.get(columns);
- DataMapWriter writer = factory.createWriter(segmentId);
+ DataMapWriter writer = factory.createWriter(new Segment(segmentId, null));
if (writers != null) {
writers.add(writer);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
index 7b1ab9d..b7270b9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
@@ -109,6 +109,11 @@ public class CarbonDataLoadConfiguration {
*/
private short writingCoresCount;
+ /**
+ * Flder path to where data should be written for this load.
+ */
+ private String dataWritePath;
+
public CarbonDataLoadConfiguration() {
}
@@ -363,4 +368,12 @@ public class CarbonDataLoadConfiguration {
public void setWritingCoresCount(short writingCoresCount) {
this.writingCoresCount = writingCoresCount;
}
+
+ public String getDataWritePath() {
+ return dataWritePath;
+ }
+
+ public void setDataWritePath(String dataWritePath) {
+ this.dataWritePath = dataWritePath;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index ba24d41..f5b29e7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -57,7 +57,8 @@ public final class DataLoadProcessBuilder {
CarbonIterator[] inputIterators) throws Exception {
CarbonDataLoadConfiguration configuration = createConfiguration(loadModel, storeLocation);
SortScopeOptions.SortScope sortScope = CarbonDataProcessorUtil.getSortScope(configuration);
- if (!configuration.isSortTable() || sortScope.equals(SortScopeOptions.SortScope.NO_SORT)) {
+ if ((!configuration.isSortTable() || sortScope.equals(SortScopeOptions.SortScope.NO_SORT))
+ && !loadModel.isPartitionLoad()) {
return buildInternalForNoSort(inputIterators, configuration);
} else if (configuration.getBucketingInfo() != null) {
return buildInternalForBucketing(inputIterators, configuration);
@@ -251,6 +252,7 @@ public final class DataLoadProcessBuilder {
configuration.setPreFetch(loadModel.isPreFetch());
configuration.setNumberOfSortColumns(carbonTable.getNumberOfSortColumns());
configuration.setNumberOfNoDictSortColumns(carbonTable.getNumberOfNoDictSortColumns());
+ configuration.setDataWritePath(loadModel.getDataWritePath());
// For partition loading always use single core as it already runs in multiple
// threads per partition
if (carbonTable.isHivePartitionTable()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
index 85eb19b..6664a2c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/MeasureFieldConverterImpl.java
@@ -89,7 +89,7 @@ public class MeasureFieldConverterImpl implements FieldConverter {
} else {
try {
if (dataField.isUseActualData()) {
- output = DataTypeUtil.getConvertedMeasureValueBasedOnDataType(value, dataType, measure);
+ output = DataTypeUtil.getMeasureValueBasedOnDataType(value, dataType, measure, true);
} else {
output = DataTypeUtil.getMeasureValueBasedOnDataType(value, dataType, measure);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index 4c536ea..a17178a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -193,6 +193,11 @@ public class CarbonLoadModel implements Serializable {
*/
private boolean isPartitionLoad;
+ /**
+ * Flder path to where data should be written for this load.
+ */
+ private String dataWritePath;
+
public boolean isAggLoadRequest() {
return isAggLoadRequest;
}
@@ -870,4 +875,12 @@ public class CarbonLoadModel implements Serializable {
public void setPartitionLoad(boolean partitionLoad) {
isPartitionLoad = partitionLoad;
}
+
+ public String getDataWritePath() {
+ return dataWritePath;
+ }
+
+ public void setDataWritePath(String dataWritePath) {
+ this.dataWritePath = dataWritePath;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index be27866..3f1430d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -26,6 +26,7 @@ import java.util.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -42,6 +43,7 @@ import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
@@ -280,7 +282,7 @@ public final class CarbonDataMergerUtil {
*/
public static boolean updateLoadMetadataWithMergeStatus(List<LoadMetadataDetails> loadsToMerge,
String metaDataFilepath, String mergedLoadNumber, CarbonLoadModel carbonLoadModel,
- CompactionType compactionType) throws IOException {
+ CompactionType compactionType, String segmentFile) throws IOException {
boolean tableStatusUpdationStatus = false;
AbsoluteTableIdentifier absoluteTableIdentifier =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
@@ -325,6 +327,7 @@ public final class CarbonDataMergerUtil {
loadMetadataDetails.setLoadEndTime(loadEnddate);
CarbonTable carbonTable = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable();
loadMetadataDetails.setLoadName(mergedLoadNumber);
+ loadMetadataDetails.setSegmentFile(segmentFile);
CarbonLoaderUtil
.addDataIndexSizeIntoMetaEntry(loadMetadataDetails, mergedLoadNumber, carbonTable);
loadMetadataDetails.setLoadStartTime(carbonLoadModel.getFactTimeStamp());
@@ -385,7 +388,7 @@ public final class CarbonDataMergerUtil {
*/
public static List<LoadMetadataDetails> identifySegmentsToBeMerged(
CarbonLoadModel carbonLoadModel, long compactionSize,
- List<LoadMetadataDetails> segments, CompactionType compactionType) {
+ List<LoadMetadataDetails> segments, CompactionType compactionType) throws IOException {
String tablePath = carbonLoadModel.getTablePath();
Map<String, String> tableLevelProperties = carbonLoadModel.getCarbonDataLoadSchema()
.getCarbonTable().getTableInfo().getFactTable().getTableProperties();
@@ -590,13 +593,13 @@ public final class CarbonDataMergerUtil {
*/
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnSize(
long compactionSize, List<LoadMetadataDetails> listOfSegmentsAfterPreserve,
- CarbonLoadModel carbonLoadModel, String tablePath) {
+ CarbonLoadModel carbonLoadModel, String tablePath) throws IOException {
List<LoadMetadataDetails> segmentsToBeMerged =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- CarbonTableIdentifier tableIdentifier =
- carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getCarbonTableIdentifier();
+ CarbonTable carbonTable = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable();
+ CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier();
// total length
@@ -612,8 +615,15 @@ public final class CarbonDataMergerUtil {
String segId = segment.getLoadName();
// variable to store one segment size across partition.
- long sizeOfOneSegmentAcrossPartition =
- getSizeOfSegment(tablePath, tableIdentifier, segId);
+ long sizeOfOneSegmentAcrossPartition;
+ if (segment.getSegmentFile() != null) {
+ CarbonTablePath carbonTablePath = CarbonStorePath
+ .getCarbonTablePath(carbonTable.getTablePath(), carbonTable.getCarbonTableIdentifier());
+ sizeOfOneSegmentAcrossPartition = CarbonUtil
+ .getSizeOfSegment(carbonTablePath, new Segment(segId, segment.getSegmentFile()));
+ } else {
+ sizeOfOneSegmentAcrossPartition = getSizeOfSegment(tablePath, tableIdentifier, segId);
+ }
// if size of a segment is greater than the Major compaction size. then ignore it.
if (sizeOfOneSegmentAcrossPartition > (compactionSize * 1024 * 1024)) {
@@ -863,18 +873,18 @@ public final class CarbonDataMergerUtil {
* @param loadMetadataDetails
* @return
*/
- public static String getValidSegments(List<LoadMetadataDetails> loadMetadataDetails) {
- StringBuilder builder = new StringBuilder();
+ public static List<Segment> getValidSegments(List<LoadMetadataDetails> loadMetadataDetails) {
+ List<Segment> segments = new ArrayList<>();
for (LoadMetadataDetails segment : loadMetadataDetails) {
//check if this load is an already merged load.
if (null != segment.getMergedLoadName()) {
- builder.append(segment.getMergedLoadName()).append(",");
+
+ segments.add(Segment.toSegment(segment.getMergedLoadName()));
} else {
- builder.append(segment.getLoadName()).append(",");
+ segments.add(Segment.toSegment(segment.getLoadName()));
}
}
- builder.deleteCharAt(builder.length() - 1);
- return builder.toString();
+ return segments;
}
/**
@@ -883,7 +893,7 @@ public final class CarbonDataMergerUtil {
* @param absoluteTableIdentifier
* @return
*/
- public static List<String> getValidSegmentList(AbsoluteTableIdentifier absoluteTableIdentifier)
+ public static List<Segment> getValidSegmentList(AbsoluteTableIdentifier absoluteTableIdentifier)
throws IOException {
SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegments = null;
@@ -933,7 +943,8 @@ public final class CarbonDataMergerUtil {
int numberUpdateDeltaFilesThreshold =
CarbonProperties.getInstance().getNoUpdateDeltaFilesThresholdForIUDCompaction();
for (LoadMetadataDetails seg : segments) {
- if ((isSegmentValid(seg)) && checkUpdateDeltaFilesInSeg(seg.getLoadName(),
+ if ((isSegmentValid(seg)) && checkUpdateDeltaFilesInSeg(
+ new Segment(seg.getLoadName(), seg.getSegmentFile()),
absoluteTableIdentifier, carbonLoadModel.getSegmentUpdateStatusManager(),
numberUpdateDeltaFilesThreshold)) {
validSegments.add(seg);
@@ -950,12 +961,12 @@ public final class CarbonDataMergerUtil {
/**
* method gets the segments list which get qualified for IUD compaction.
- * @param Segments
+ * @param segments
* @param absoluteTableIdentifier
* @param compactionTypeIUD
* @return
*/
- public static List<String> getSegListIUDCompactionQualified(List<String> Segments,
+ public static List<String> getSegListIUDCompactionQualified(List<Segment> segments,
AbsoluteTableIdentifier absoluteTableIdentifier,
SegmentUpdateStatusManager segmentUpdateStatusManager, CompactionType compactionTypeIUD) {
@@ -964,8 +975,8 @@ public final class CarbonDataMergerUtil {
if (CompactionType.IUD_DELETE_DELTA == compactionTypeIUD) {
int numberDeleteDeltaFilesThreshold =
CarbonProperties.getInstance().getNoDeleteDeltaFilesThresholdForIUDCompaction();
- List<String> deleteSegments = new ArrayList<>();
- for (String seg : Segments) {
+ List<Segment> deleteSegments = new ArrayList<>();
+ for (Segment seg : segments) {
if (checkDeleteDeltaFilesInSeg(seg, segmentUpdateStatusManager,
numberDeleteDeltaFilesThreshold)) {
deleteSegments.add(seg);
@@ -975,7 +986,7 @@ public final class CarbonDataMergerUtil {
// This Code Block Append the Segname along with the Blocks selected for Merge instead of
// only taking the segment name. This will help to parallelize better for each block
// in case of Delete Horizontal Compaction.
- for (String segName : deleteSegments) {
+ for (Segment segName : deleteSegments) {
List<String> tempSegments = getDeleteDeltaFilesInSeg(segName, segmentUpdateStatusManager,
numberDeleteDeltaFilesThreshold);
validSegments.addAll(tempSegments);
@@ -984,10 +995,10 @@ public final class CarbonDataMergerUtil {
} else if (CompactionType.IUD_UPDDEL_DELTA == compactionTypeIUD) {
int numberUpdateDeltaFilesThreshold =
CarbonProperties.getInstance().getNoUpdateDeltaFilesThresholdForIUDCompaction();
- for (String seg : Segments) {
+ for (Segment seg : segments) {
if (checkUpdateDeltaFilesInSeg(seg, absoluteTableIdentifier, segmentUpdateStatusManager,
numberUpdateDeltaFilesThreshold)) {
- validSegments.add(seg);
+ validSegments.add(seg.getSegmentNo());
}
}
}
@@ -1027,7 +1038,7 @@ public final class CarbonDataMergerUtil {
* @param numberDeltaFilesThreshold
* @return
*/
- public static Boolean checkUpdateDeltaFilesInSeg(String seg,
+ public static Boolean checkUpdateDeltaFilesInSeg(Segment seg,
AbsoluteTableIdentifier absoluteTableIdentifier,
SegmentUpdateStatusManager segmentUpdateStatusManager, int numberDeltaFilesThreshold) {
@@ -1036,14 +1047,14 @@ public final class CarbonDataMergerUtil {
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", seg);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", seg.getSegmentNo());
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
updateDeltaFiles = segmentUpdateStatusManager
- .getUpdateDeltaFilesForSegment(seg, true, CarbonCommonConstants.UPDATE_DELTA_FILE_EXT,
- false, allSegmentFiles);
+ .getUpdateDeltaFilesForSegment(seg.getSegmentNo(), true,
+ CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, false, allSegmentFiles);
if (updateDeltaFiles == null) {
return false;
@@ -1079,11 +1090,12 @@ public final class CarbonDataMergerUtil {
* @param numberDeltaFilesThreshold
* @return
*/
- private static boolean checkDeleteDeltaFilesInSeg(String seg,
+ private static boolean checkDeleteDeltaFilesInSeg(Segment seg,
SegmentUpdateStatusManager segmentUpdateStatusManager, int numberDeltaFilesThreshold) {
Set<String> uniqueBlocks = new HashSet<String>();
- List<String> blockNameList = segmentUpdateStatusManager.getBlockNameFromSegment(seg);
+ List<String> blockNameList =
+ segmentUpdateStatusManager.getBlockNameFromSegment(seg.getSegmentNo());
for (final String blockName : blockNameList) {
@@ -1121,11 +1133,12 @@ public final class CarbonDataMergerUtil {
* @return
*/
- private static List<String> getDeleteDeltaFilesInSeg(String seg,
+ private static List<String> getDeleteDeltaFilesInSeg(Segment seg,
SegmentUpdateStatusManager segmentUpdateStatusManager, int numberDeltaFilesThreshold) {
List<String> blockLists = new ArrayList<>();
- List<String> blockNameList = segmentUpdateStatusManager.getBlockNameFromSegment(seg);
+ List<String> blockNameList =
+ segmentUpdateStatusManager.getBlockNameFromSegment(seg.getSegmentNo());
for (final String blockName : blockNameList) {
@@ -1133,7 +1146,7 @@ public final class CarbonDataMergerUtil {
segmentUpdateStatusManager.getDeleteDeltaFilesList(seg, blockName);
if (deleteDeltaFiles.length > numberDeltaFilesThreshold) {
- blockLists.add(seg + "/" + blockName);
+ blockLists.add(seg.getSegmentNo() + "/" + blockName);
}
}
return blockLists;
@@ -1177,7 +1190,7 @@ public final class CarbonDataMergerUtil {
segmentUpdateStatusManager.setUpdateStatusDetails(segmentUpdateDetails);
CarbonFile[] deleteDeltaFiles =
- segmentUpdateStatusManager.getDeleteDeltaFilesList(seg, blockName);
+ segmentUpdateStatusManager.getDeleteDeltaFilesList(new Segment(seg, null), blockName);
String destFileName =
blockName + "-" + timestamp.toString() + CarbonCommonConstants.DELETE_DELTA_FILE_EXT;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 2480a39..2fbdf4f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -26,7 +26,8 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
-import org.apache.carbondata.core.metadata.PartitionMapFileStore;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -35,7 +36,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
@@ -129,19 +129,20 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
*/
private SortIntermediateFileMerger intermediateFileMerger;
- private List<String> partitionNames;
+ private PartitionSpec partitionSpec;
+
private SortParameters sortParameters;
public CompactionResultSortProcessor(CarbonLoadModel carbonLoadModel, CarbonTable carbonTable,
SegmentProperties segmentProperties, CompactionType compactionType, String tableName,
- List<String> partitionNames) {
+ PartitionSpec partitionSpec) {
this.carbonLoadModel = carbonLoadModel;
this.carbonTable = carbonTable;
this.segmentProperties = segmentProperties;
this.segmentId = carbonLoadModel.getSegmentId();
this.compactionType = compactionType;
this.tableName = tableName;
- this.partitionNames = partitionNames;
+ this.partitionSpec = partitionSpec;
}
/**
@@ -168,14 +169,12 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
} catch (Exception e) {
LOGGER.error(e, "Compaction failed: " + e.getMessage());
} finally {
- if (partitionNames != null) {
+ if (partitionSpec != null) {
try {
- new PartitionMapFileStore().writePartitionMapFile(
- CarbonTablePath.getSegmentPath(
- carbonLoadModel.getTablePath(),
- carbonLoadModel.getSegmentId()),
- carbonLoadModel.getTaskNo(),
- partitionNames);
+ SegmentFileStore
+ .writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(),
+ partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "",
+ partitionSpec.getPartitions());
} catch (IOException e) {
LOGGER.error(e, "Compaction failed: " + e.getMessage());
isCompactionSuccess = false;
@@ -401,9 +400,19 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
* initialise carbon data writer instance
*/
private void initDataHandler() throws Exception {
+ String carbonStoreLocation;
+ if (partitionSpec != null) {
+ carbonStoreLocation =
+ partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR
+ + carbonLoadModel.getFactTimeStamp() + ".tmp";
+ } else {
+ carbonStoreLocation = CarbonDataProcessorUtil
+ .createCarbonStoreLocation(carbonTable.getTablePath(), carbonLoadModel.getDatabaseName(),
+ tableName, carbonLoadModel.getPartitionId(), carbonLoadModel.getSegmentId());
+ }
CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
.getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,
- tempStoreLocation);
+ tempStoreLocation, carbonStoreLocation);
setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonTable,
carbonFactDataHandlerModel);
dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
index 3d0700b..b41829f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
@@ -24,18 +24,19 @@ import java.util.PriorityQueue;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.PartitionMapFileStore;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.exception.SliceMergerException;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerColumnar;
@@ -51,7 +52,7 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
private CarbonFactHandler dataHandler;
private SegmentProperties segprop;
private CarbonLoadModel loadModel;
- private List<String> partitionNames;
+ private PartitionSpec partitionSpec;
/**
* record holder heap
*/
@@ -62,16 +63,26 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
public RowResultMergerProcessor(String databaseName,
String tableName, SegmentProperties segProp, String[] tempStoreLocation,
- CarbonLoadModel loadModel, CompactionType compactionType, List<String> partitionNames) {
+ CarbonLoadModel loadModel, CompactionType compactionType, PartitionSpec partitionSpec) {
this.segprop = segProp;
- this.partitionNames = partitionNames;
+ this.partitionSpec = partitionSpec;
this.loadModel = loadModel;
CarbonDataProcessorUtil.createLocations(tempStoreLocation);
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName);
+ String carbonStoreLocation;
+ if (partitionSpec != null) {
+ carbonStoreLocation =
+ partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR + loadModel
+ .getFactTimeStamp() + ".tmp";
+ } else {
+ carbonStoreLocation = CarbonDataProcessorUtil
+ .createCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(),
+ tableName, loadModel.getPartitionId(), loadModel.getSegmentId());
+ }
CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
.getCarbonFactDataHandlerModel(loadModel, carbonTable, segProp, tableName,
- tempStoreLocation);
+ tempStoreLocation, carbonStoreLocation);
setDataFileAttributesInModel(loadModel, compactionType, carbonTable,
carbonFactDataHandlerModel);
carbonFactDataHandlerModel.setCompactionFlow(true);
@@ -157,14 +168,13 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
if (isDataPresent) {
this.dataHandler.closeHandler();
}
- if (partitionNames != null) {
- new PartitionMapFileStore().writePartitionMapFile(
- CarbonTablePath.getSegmentPath(loadModel.getTablePath(), loadModel.getSegmentId()),
- loadModel.getTaskNo(),
- partitionNames);
+ if (partitionSpec != null) {
+ SegmentFileStore.writeSegmentFile(loadModel.getTablePath(), loadModel.getTaskNo(),
+ partitionSpec.getLocation().toString(), loadModel.getFactTimeStamp() + "",
+ partitionSpec.getPartitions());
}
} catch (CarbonDataWriterException | IOException e) {
- LOGGER.error(e,"Exception in compaction merger");
+ LOGGER.error(e, "Exception in compaction merger");
mergeStatus = false;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
index 68a212e..ff6ca93 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
@@ -47,9 +47,12 @@ public class RowResultProcessor {
CarbonDataProcessorUtil.createLocations(tempStoreLocation);
this.segmentProperties = segProp;
String tableName = carbonTable.getTableName();
+ String carbonStoreLocation = CarbonDataProcessorUtil
+ .createCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(),
+ tableName, loadModel.getPartitionId(), loadModel.getSegmentId());
CarbonFactDataHandlerModel carbonFactDataHandlerModel =
CarbonFactDataHandlerModel.getCarbonFactDataHandlerModel(loadModel, carbonTable,
- segProp, tableName, tempStoreLocation);
+ segProp, tableName, tempStoreLocation, carbonStoreLocation);
CarbonDataFileAttributes carbonDataFileAttributes =
new CarbonDataFileAttributes(Long.parseLong(loadModel.getTaskNo()),
loadModel.getFactTimeStamp());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index d15152c..d77fcab 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -275,7 +275,7 @@ public class CarbonFactDataHandlerModel {
*/
public static CarbonFactDataHandlerModel getCarbonFactDataHandlerModel(CarbonLoadModel loadModel,
CarbonTable carbonTable, SegmentProperties segmentProperties, String tableName,
- String[] tempStoreLocation) {
+ String[] tempStoreLocation, String carbonDataDirectoryPath) {
CarbonFactDataHandlerModel carbonFactDataHandlerModel = new CarbonFactDataHandlerModel();
carbonFactDataHandlerModel.setSchemaUpdatedTimeStamp(carbonTable.getTableLastUpdatedTime());
carbonFactDataHandlerModel.setDatabaseName(loadModel.getDatabaseName());
@@ -307,9 +307,7 @@ public class CarbonFactDataHandlerModel {
measureDataTypes[i++] = msr.getDataType();
}
carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes);
- String carbonDataDirectoryPath = CarbonDataProcessorUtil
- .checkAndCreateCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(),
- tableName, loadModel.getPartitionId(), loadModel.getSegmentId());
+ CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
List<CarbonDimension> dimensionByTableName = carbonTable.getDimensionByTableName(tableName);
boolean[] isUseInvertedIndexes = new boolean[dimensionByTableName.size()];
@@ -334,6 +332,10 @@ public class CarbonFactDataHandlerModel {
* @return data directory path
*/
private static String getCarbonDataFolderLocation(CarbonDataLoadConfiguration configuration) {
+ if (configuration.getDataWritePath() != null) {
+ CarbonUtil.checkAndCreateFolder(configuration.getDataWritePath());
+ return configuration.getDataWritePath();
+ }
AbsoluteTableIdentifier absoluteTableIdentifier = configuration.getTableIdentifier();
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
String carbonDataDirectoryPath = carbonTablePath
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 376a546..1e648e1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -302,7 +302,7 @@ public final class CarbonDataProcessorUtil {
}
public static boolean isHeaderValid(String tableName, String[] csvHeader,
- CarbonDataLoadSchema schema) {
+ CarbonDataLoadSchema schema, List<String> ignoreColumns) {
Iterator<String> columnIterator =
CarbonDataProcessorUtil.getSchemaColumnNames(schema, tableName).iterator();
Set<String> csvColumns = new HashSet<String>(csvHeader.length);
@@ -311,7 +311,8 @@ public final class CarbonDataProcessorUtil {
// file header should contain all columns of carbon table.
// So csvColumns should contain all elements of columnIterator.
while (columnIterator.hasNext()) {
- if (!csvColumns.contains(columnIterator.next().toLowerCase())) {
+ String column = columnIterator.next().toLowerCase();
+ if (!csvColumns.contains(column) && !ignoreColumns.contains(column)) {
return false;
}
}
@@ -377,7 +378,7 @@ public final class CarbonDataProcessorUtil {
*
* @return data directory path
*/
- public static String checkAndCreateCarbonStoreLocation(String factStoreLocation,
+ public static String createCarbonStoreLocation(String factStoreLocation,
String databaseName, String tableName, String partitionId, String segmentId) {
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName);
CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
@@ -385,7 +386,6 @@ public final class CarbonDataProcessorUtil {
CarbonStorePath.getCarbonTablePath(factStoreLocation, carbonTableIdentifier);
String carbonDataDirectoryPath =
carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId);
- CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
return carbonDataDirectoryPath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 00f13a5..32c72da 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.block.Distributable;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
@@ -166,6 +167,23 @@ public final class CarbonLoaderUtil {
public static boolean recordNewLoadMetadata(LoadMetadataDetails newMetaEntry,
CarbonLoadModel loadModel, boolean loadStartEntry, boolean insertOverwrite, String uuid)
throws IOException {
+ return recordNewLoadMetadata(newMetaEntry, loadModel, loadStartEntry, insertOverwrite, uuid,
+ new ArrayList<Segment>(), new ArrayList<Segment>());
+ }
+
+ /**
+ * This API will write the load level metadata for the loadmanagement module inorder to
+ * manage the load and query execution management smoothly.
+ *
+ * @param newMetaEntry
+ * @param loadModel
+ * @param uuid
+ * @return boolean which determines whether status update is done or not.
+ * @throws IOException
+ */
+ public static boolean recordNewLoadMetadata(LoadMetadataDetails newMetaEntry,
+ CarbonLoadModel loadModel, boolean loadStartEntry, boolean insertOverwrite, String uuid,
+ List<Segment> segmentsToBeDeleted, List<Segment> segmentFilesTobeUpdated) throws IOException {
boolean status = false;
AbsoluteTableIdentifier absoluteTableIdentifier =
loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
@@ -237,9 +255,11 @@ public final class CarbonLoaderUtil {
// existing entry needs to be overwritten as the entry will exist with some
// intermediate status
int indexToOverwriteNewMetaEntry = 0;
+ boolean found = false;
for (LoadMetadataDetails entry : listOfLoadFolderDetails) {
if (entry.getLoadName().equals(newMetaEntry.getLoadName())
&& entry.getLoadStartTime() == newMetaEntry.getLoadStartTime()) {
+ found = true;
break;
}
indexToOverwriteNewMetaEntry++;
@@ -254,6 +274,10 @@ public final class CarbonLoaderUtil {
}
}
}
+ if (!found) {
+ LOGGER.error("Entry not found to update " + newMetaEntry + " From list :: "
+ + listOfLoadFolderDetails);
+ }
listOfLoadFolderDetails.set(indexToOverwriteNewMetaEntry, newMetaEntry);
}
// when no records are inserted then newSegmentEntry will be SegmentStatus.MARKED_FOR_DELETE
@@ -262,6 +286,17 @@ public final class CarbonLoaderUtil {
addToStaleFolders(carbonTablePath, staleFolders, newMetaEntry);
}
+ for (LoadMetadataDetails detail: listOfLoadFolderDetails) {
+ // if the segments is in the list of marked for delete then update the status.
+ if (segmentsToBeDeleted.contains(new Segment(detail.getLoadName(), null))) {
+ detail.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
+ } else if (segmentFilesTobeUpdated.contains(Segment.toSegment(detail.getLoadName()))) {
+ detail.setSegmentFile(
+ detail.getLoadName() + "_" + newMetaEntry.getUpdateStatusFileName()
+ + CarbonTablePath.SEGMENT_EXT);
+ }
+ }
+
SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetails
.toArray(new LoadMetadataDetails[listOfLoadFolderDetails.size()]));
// Delete all old stale segment folders
@@ -907,8 +942,8 @@ public final class CarbonLoaderUtil {
String segmentId, CarbonTable carbonTable) throws IOException {
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath((carbonTable.getAbsoluteTableIdentifier()));
- Map<String, Long> dataIndexSize =
- CarbonUtil.getDataSizeAndIndexSize(carbonTablePath, segmentId);
+ Map<String, Long> dataIndexSize = CarbonUtil.getDataSizeAndIndexSize(carbonTablePath,
+ new Segment(segmentId, loadMetadataDetails.getSegmentFile()));
Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
loadMetadataDetails.setDataSize(String.valueOf(dataSize));
Long indexSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
index 02ab1d8..52b9f52 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
@@ -18,16 +18,19 @@
package org.apache.carbondata.processing.util;
import java.io.IOException;
+import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.locks.CarbonLockFactory;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.locks.LockUsage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
@@ -60,54 +63,61 @@ public final class DeleteLoadFolders {
}
public static void physicalFactAndMeasureMetadataDeletion(
- AbsoluteTableIdentifier absoluteTableIdentifier, String metadataPath, boolean isForceDelete) {
+ AbsoluteTableIdentifier absoluteTableIdentifier, String metadataPath, boolean isForceDelete,
+ List<PartitionSpec> specs) {
LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
for (LoadMetadataDetails oneLoad : currentDetails) {
if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) {
- String path = getSegmentPath(absoluteTableIdentifier, 0, oneLoad);
- boolean status = false;
try {
- if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
- CarbonFile file = FileFactory.getCarbonFile(path, FileFactory.getFileType(path));
- CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {
-
- @Override public boolean accept(CarbonFile file) {
- return (CarbonTablePath.isCarbonDataFile(file.getName())
- || CarbonTablePath.isCarbonIndexFile(file.getName())
- || CarbonTablePath.isPartitionMapFile(file.getName()));
- }
- });
+ if (oneLoad.getSegmentFile() != null) {
+ SegmentFileStore
+ .deleteSegment(absoluteTableIdentifier.getTablePath(), oneLoad.getSegmentFile(),
+ specs);
+ } else {
+ String path = getSegmentPath(absoluteTableIdentifier, 0, oneLoad);
+ boolean status = false;
+ if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
+ CarbonFile file = FileFactory.getCarbonFile(path, FileFactory.getFileType(path));
+ CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {
+
+ @Override public boolean accept(CarbonFile file) {
+ return (CarbonTablePath.isCarbonDataFile(file.getName()) ||
+ CarbonTablePath.isCarbonIndexFile(file.getName()));
+ }
+ });
- //if there are no fact and msr metadata files present then no need to keep
- //entry in metadata.
- if (filesToBeDeleted.length == 0) {
- status = true;
- } else {
+ //if there are no fact and msr metadata files present then no need to keep
+ //entry in metadata.
+ if (filesToBeDeleted.length == 0) {
+ status = true;
+ } else {
- for (CarbonFile eachFile : filesToBeDeleted) {
- if (!eachFile.delete()) {
- LOGGER.warn("Unable to delete the file as per delete command " + eachFile
- .getAbsolutePath());
- status = false;
- } else {
- status = true;
+ for (CarbonFile eachFile : filesToBeDeleted) {
+ if (!eachFile.delete()) {
+ LOGGER.warn("Unable to delete the file as per delete command " + eachFile
+ .getAbsolutePath());
+ status = false;
+ } else {
+ status = true;
+ }
}
}
- }
- // need to delete the complete folder.
- if (status) {
- if (!file.delete()) {
- LOGGER.warn(
- "Unable to delete the folder as per delete command " + file.getAbsolutePath());
+ // need to delete the complete folder.
+ if (status) {
+ if (!file.delete()) {
+ LOGGER.warn("Unable to delete the folder as per delete command " + file
+ .getAbsolutePath());
+ }
}
+
+ } else {
+ LOGGER.warn("Files are not found in segment " + path
+ + " it seems, files are already being deleted");
}
- } else {
- LOGGER.warn("Files are not found in segment " + path
- + " it seems, files are already being deleted");
}
} catch (IOException e) {
- LOGGER.warn("Unable to delete the file as per delete command " + path);
+ LOGGER.warn("Unable to delete the file as per delete command " + oneLoad.getLoadName());
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
index 63320ef..cd1e28a 100644
--- a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
+++ b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
@@ -83,7 +83,7 @@ public class BlockIndexStoreTest extends TestCase {
// assertTrue(false);
// }
// List<String> segmentIds = new ArrayList<>();
-// segmentIds.add(info.getSegmentId());
+// segmentIds.add(info.getSegment());
// cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
// }
//
@@ -151,7 +151,7 @@ public class BlockIndexStoreTest extends TestCase {
// }
// List<String> segmentIds = new ArrayList<>();
// for (TableBlockInfo tableBlockInfo : tableBlockInfos) {
-// segmentIds.add(tableBlockInfo.getSegmentId());
+// segmentIds.add(tableBlockInfo.getSegment());
// }
// cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
// }
@@ -223,7 +223,7 @@ public class BlockIndexStoreTest extends TestCase {
// }
// List<String> segmentIds = new ArrayList<>();
// for (TableBlockInfo tableBlockInfo : tableBlockInfos) {
-// segmentIds.add(tableBlockInfo.getSegmentId());
+// segmentIds.add(tableBlockInfo.getSegment());
// }
// cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
// }
[02/16] carbondata git commit: [CARBONDATA-2151][Streaming] Fix
filter query issue on streaming table
Posted by gv...@apache.org.
[CARBONDATA-2151][Streaming] Fix filter query issue on streaming table
1.Fix filter query issue for timestamp, date, decimal
2.Add more test case
dataType: int, streaming, float, double, decimal, timestamp, date, complex
operation: =, <>, >=, >, <, <=, in, like, between, is null, is not null
This closes #1969
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f8a62a9b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f8a62a9b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f8a62a9b
Branch: refs/heads/branch-1.3
Commit: f8a62a9bd8ba39cd6bc247c587a7a3e1afd99254
Parents: c2785b3
Author: QiangCai <qi...@qq.com>
Authored: Sun Feb 11 16:06:01 2018 +0800
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Tue Feb 27 12:46:13 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/scan/filter/FilterUtil.java | 4 +
.../executer/ExcludeFilterExecuterImpl.java | 2 +-
.../scan/filter/executer/FilterExecuter.java | 4 +
.../executer/RangeValueFilterExecuterImpl.java | 29 +
.../executer/RowLevelFilterExecuterImpl.java | 3 +-
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 18 +
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 18 +
...velRangeLessThanEqualFilterExecuterImpl.java | 18 +
.../RowLevelRangeLessThanFiterExecuterImpl.java | 18 +
.../streaming/CarbonStreamRecordReader.java | 2 +-
.../src/test/resources/streamSample.csv | 12 +-
.../TestStreamingTableOperation.scala | 547 +++++++++++++++++--
12 files changed, 612 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 689da9f..8dcac30 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -1790,6 +1790,10 @@ public final class FilterUtil {
}
public static void updateIndexOfColumnExpression(Expression exp, int dimOridnalMax) {
+ // if expression is null, not require to update index.
+ if (exp == null) {
+ return;
+ }
if (exp.getChildren() == null || exp.getChildren().size() == 0) {
if (exp instanceof ColumnExpression) {
ColumnExpression ce = (ColumnExpression) exp;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 465bee6..fad37fc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -135,7 +135,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
@Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
if (isDimensionPresentInCurrentBlock) {
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
byte[] col = (byte[])value.getVal(dimColEvaluatorInfo.getDimension().getOrdinal());
for (int i = 0; i < filterValues.length; i++) {
if (0 == ByteUtil.UnsafeComparer.INSTANCE.compareTo(col, 0, col.length,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
index 85891dc..53d3068 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
@@ -35,6 +35,10 @@ public interface FilterExecuter {
BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException;
+ /**
+ * apply range filter on a row
+ * @return true: if the value satisfy the filter; or else false.
+ */
boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index ee373c5..797fe9d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.scan.expression.conditional.LessThanEqualToExp
import org.apache.carbondata.core.scan.expression.conditional.LessThanExpression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
@@ -146,6 +147,34 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
/**
+ * apply range filter on a row
+ */
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ throws FilterUnsupportedException, IOException {
+
+ byte[] col = (byte[]) value.getVal(dimColEvaluatorInfo.getDimension().getOrdinal());
+ byte[][] filterValues = this.filterRangesValues;
+
+ if (isDimensionPresentInCurrentBlock) {
+ boolean result;
+ if (greaterThanExp) {
+ result = ByteUtil.compare(filterValues[0], col) < 0;
+ } else {
+ result = ByteUtil.compare(filterValues[0], col) <= 0;
+ }
+
+ if (result) {
+ if (lessThanExp) {
+ return ByteUtil.compare(filterValues[1], col) > 0;
+ } else {
+ return ByteUtil.compare(filterValues[1], col) >= 0;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
* Method to find presence of LessThan Expression.
* @return
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 89489a2..8956f30 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -272,7 +272,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
return bitSetGroup;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
try {
return exp.evaluate(value).getBoolean();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 306f3fa..3981211 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
@@ -248,6 +249,23 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return null;
}
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ throws FilterUnsupportedException, IOException {
+ if (isDimensionPresentInCurrentBlock[0]) {
+ byte[] col =
+ (byte[]) value.getVal(dimColEvaluatorInfoList.get(0).getDimension().getOrdinal());
+ return ByteUtil.compare(filterRangeValues[0], col) < 0;
+ }
+
+ if (isMeasurePresentInCurrentBlock[0]) {
+ Object col =
+ value.getVal(msrColEvalutorInfoList.get(0).getMeasure().getOrdinal() + dimOrdinalMax);
+ return comparator.compare(msrFilterRangeValues[0], col) < 0;
+ }
+ return false;
+ }
+
private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index db55e42..f2ddcb6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
@@ -248,6 +249,23 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
return null;
}
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ throws FilterUnsupportedException, IOException {
+ if (isDimensionPresentInCurrentBlock[0]) {
+ byte[] col =
+ (byte[]) value.getVal(dimColEvaluatorInfoList.get(0).getDimension().getOrdinal());
+ return ByteUtil.compare(filterRangeValues[0], col) <= 0;
+ }
+
+ if (isMeasurePresentInCurrentBlock[0]) {
+ Object col =
+ value.getVal(msrColEvalutorInfoList.get(0).getMeasure().getOrdinal() + dimOrdinalMax);
+ return comparator.compare(msrFilterRangeValues[0], col) <= 0;
+ }
+ return false;
+ }
+
private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 88cf75c..a44bc1a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
@@ -230,6 +231,23 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return null;
}
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ throws FilterUnsupportedException, IOException {
+ if (isDimensionPresentInCurrentBlock[0]) {
+ byte[] col =
+ (byte[]) value.getVal(dimColEvaluatorInfoList.get(0).getDimension().getOrdinal());
+ return ByteUtil.compare(filterRangeValues[0], col) >= 0;
+ }
+
+ if (isMeasurePresentInCurrentBlock[0]) {
+ Object col =
+ value.getVal(msrColEvalutorInfoList.get(0).getMeasure().getOrdinal() + dimOrdinalMax);
+ return comparator.compare(msrFilterRangeValues[0], col) >= 0;
+ }
+ return false;
+ }
+
private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 7f735c2..447ab46 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
@@ -232,6 +233,23 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return null;
}
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ throws FilterUnsupportedException, IOException {
+ if (isDimensionPresentInCurrentBlock[0]) {
+ byte[] col =
+ (byte[]) value.getVal(dimColEvaluatorInfoList.get(0).getDimension().getOrdinal());
+ return ByteUtil.compare(filterRangeValues[0], col) > 0;
+ }
+
+ if (isMeasurePresentInCurrentBlock[0]) {
+ Object col =
+ value.getVal(msrColEvalutorInfoList.get(0).getMeasure().getOrdinal() + dimOrdinalMax);
+ return comparator.compare(msrFilterRangeValues[0], col) > 0;
+ }
+ return false;
+ }
+
private BitSet getFilteredIndexesForMeasures(ColumnPage columnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
index 19626f0..773089b 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
@@ -607,7 +607,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
filterValues[filterMap[colCount]] = v;
}
if (isProjectionRequired[colCount]) {
- outputValues[projectionMap[colCount]] = v;
+ outputValues[projectionMap[colCount]] = Decimal.apply(v);
}
} else {
input.skipBytes(len);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/integration/spark-common-test/src/test/resources/streamSample.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/streamSample.csv b/integration/spark-common-test/src/test/resources/streamSample.csv
index 590ea90..3443237 100644
--- a/integration/spark-common-test/src/test/resources/streamSample.csv
+++ b/integration/spark-common-test/src/test/resources/streamSample.csv
@@ -1,6 +1,6 @@
-id,name,city,salary,file
-100000001,batch_1,city_1,0.1,school_1:school_11$20
-100000002,batch_2,city_2,0.2,school_2:school_22$30
-100000003,batch_3,city_3,0.3,school_3:school_33$40
-100000004,batch_4,city_4,0.4,school_4:school_44$50
-100000005,batch_5,city_5,0.5,school_5:school_55$60
+id,name,city,salary,tax,percent,birthday,register,updated,file
+100000001,batch_1,city_1,0.1,0.01,80.01,1990-01-01,2010-01-01 10:01:01,2010-01-01 10:01:01,school_1:school_11$20
+100000002,batch_2,city_2,0.2,0.02,80.02,1990-01-02,2010-01-02 10:01:01,2010-01-02 10:01:01,school_2:school_22$30
+100000003,batch_3,city_3,0.3,0.03,80.03,1990-01-03,2010-01-03 10:01:01,2010-01-03 10:01:01,school_3:school_33$40
+100000004,batch_4,city_4,0.4,0.04,80.04,1990-01-04,2010-01-04 10:01:01,2010-01-04 10:01:01,school_4:school_44$50
+100000005,batch_5,city_5,0.5,0.05,80.05,1990-01-05,2010-01-05 10:01:01,2010-01-05 10:01:01,school_5:school_55$60
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f8a62a9b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 4b3a957..94baf86 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -18,7 +18,9 @@
package org.apache.spark.carbondata
import java.io.{File, PrintWriter}
+import java.math.BigDecimal
import java.net.{BindException, ServerSocket}
+import java.sql.{Date, Timestamp}
import java.util.concurrent.Executors
import scala.collection.mutable
@@ -28,14 +30,13 @@ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.{CarbonEnv, Row, SaveMode, SparkSession}
import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.types.StructType
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.statusmanager.{FileFormat, SegmentStatus}
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, ProcessMetaDataException}
-import org.apache.carbondata.streaming.CarbonStreamException
class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
@@ -43,6 +44,12 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
private val dataFilePath = s"$resourcesPath/streamSample.csv"
override def beforeAll {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_DATE_FORMAT,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
sql("DROP DATABASE IF EXISTS streaming CASCADE")
sql("CREATE DATABASE streaming")
sql("USE streaming")
@@ -235,7 +242,8 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
)
val row = sql("select * from streaming.stream_table_file order by id").head()
- assertResult(Row(10, "name_10", "city_10", 100000.0))(row)
+ val exceptedRow = Row(10, "name_10", "city_10", 100000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))
+ assertResult(exceptedRow)(row)
}
// bad records
@@ -287,12 +295,12 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
)
// non-filter
- val result = sql("select * from streaming.stream_table_filter order by id").collect()
+ val result = sql("select * from streaming.stream_table_filter order by id, name").collect()
assert(result != null)
assert(result.length == 55)
// check one row of streaming data
- assert(result(1).getInt(0) == 1)
- assert(result(1).getString(1) == "name_1")
+ assert(result(1).isNullAt(0))
+ assert(result(1).getString(1) == "name_6")
// check one row of batch loading
assert(result(50).getInt(0) == 100000001)
assert(result(50).getString(1) == "batch_1")
@@ -300,35 +308,259 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
// filter
checkAnswer(
sql("select * from stream_table_filter where id = 1"),
- Seq(Row(1, "name_1", "city_1", 10000.0)))
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
checkAnswer(
- sql("select * from stream_table_filter where name = 'name_2'"),
- Seq(Row(2, "name_2", "", 20000.0)))
+ sql("select * from stream_table_filter where id > 49 and id < 100000002"),
+ Seq(Row(50, "name_50", "city_50", 500000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0")),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id between 50 and 100000001"),
+ Seq(Row(50, "name_50", "city_50", 500000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0")),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where name in ('name_9','name_10', 'name_11', 'name_12') and id <> 10 and id not in (11, 12)"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where name = 'name_3'"),
+ Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where name like '%me_3%' and id < 30"),
+ Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where name >= 'name_3' and id < 4"),
+ Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id in (9, 10, 11, 12) and name <> 'name_10' and name not in ('name_11', 'name_12')"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
checkAnswer(
sql("select * from stream_table_filter where city = 'city_1'"),
- Seq(Row(1, "name_1", "city_1", 10000.0),
- Row(100000001, "batch_1", "city_1", 0.1)))
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0")),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
checkAnswer(
- sql("select * from stream_table_filter where id > 49 and id < 100000002"),
- Seq(Row(50, "name_50", "city_50", 500000.0),
- Row(100000001, "batch_1", "city_1", 0.1)))
+ sql("select * from stream_table_filter where city like '%ty_1%' and ( id < 10 or id >= 100000001)"),
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0")),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where city > 'city_09' and city < 'city_10'"),
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0")),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where city between 'city_09' and 'city_1'"),
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0")),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id in (9, 10, 11, 12) and city <> 'city_10' and city not in ('city_11', 'city_12')"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where salary = 90000"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where salary > 80000 and salary <= 100000"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(10, "name_10", "city_10", 100000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where salary between 80001 and 90000"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id in (9, 10, 11, 12) and salary <> 100000.0 and salary not in (110000.0, 120000.0)"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where tax = 0.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where tax >= 0.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where tax < 0.05 and tax > 0.02 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where tax between 0.02 and 0.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id in (9, 10) and tax <> 0.01"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where percent = 80.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where percent >= 80.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where percent < 80.05 and percent > 80.02 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where percent between 80.02 and 80.05 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id in (9, 10) and percent <> 80.01"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
checkAnswer(
- sql("select * from stream_table_filter where id is null"),
- Seq(Row(null, "name_6", "city_6", 60000.0)))
+ sql("select * from stream_table_filter where birthday between '1990-01-04' and '1990-01-05'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000005, "batch_5", "city_5", 0.5, BigDecimal.valueOf(0.05), 80.05, Date.valueOf("1990-01-05"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Timestamp.valueOf("2010-01-05 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where birthday = '1990-01-04'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where birthday > '1990-01-03' and birthday <= '1990-01-04'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where birthday between '1990-01-04' and '1990-01-05'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000005, "batch_5", "city_5", 0.5, BigDecimal.valueOf(0.05), 80.05, Date.valueOf("1990-01-05"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Timestamp.valueOf("2010-01-05 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id in (9, 10) and birthday <> '1990-01-01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where register = '2010-01-04 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where register > '2010-01-03 10:01:01' and register <= '2010-01-04 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where register between '2010-01-04 10:01:01' and '2010-01-05 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000005, "batch_5", "city_5", 0.5, BigDecimal.valueOf(0.05), 80.05, Date.valueOf("1990-01-05"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Timestamp.valueOf("2010-01-05 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id in (9, 10) and register <> '2010-01-01 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where updated = '2010-01-04 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where updated > '2010-01-03 10:01:01' and register <= '2010-01-04 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where updated between '2010-01-04 10:01:01' and '2010-01-05 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0")),
+ Row(100000005, "batch_5", "city_5", 0.5, BigDecimal.valueOf(0.05), 80.05, Date.valueOf("1990-01-05"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Timestamp.valueOf("2010-01-05 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id in (9, 10) and updated <> '2010-01-01 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id is null order by name"),
+ Seq(Row(null, "", "", null, null, null, null, null, null),
+ Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where name = ''"),
+ Seq(Row(null, "", "", null, null, null, null, null, null)))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id is null and name <> ''"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
checkAnswer(
sql("select * from stream_table_filter where city = ''"),
- Seq(Row(2, "name_2", "", 20000.0)))
+ Seq(Row(null, "", "", null, null, null, null, null, null)))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id is null and city <> ''"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where salary is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null)))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id is null and salary is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where tax is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null)))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id is null and tax is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where percent is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null)))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id is null and percent is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where birthday is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null)))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id is null and birthday is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where register is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null)))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id is null and register is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where updated is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null)))
+
+ checkAnswer(
+ sql("select * from stream_table_filter where id is null and updated is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"))))
// agg
checkAnswer(
sql("select count(*), max(id), min(name), cast(avg(id) as integer), sum(id) " +
"from stream_table_filter where id >= 2 and id <= 100000004"),
- Seq(Row(52, 100000004, "batch_1", 7692332, 400001278)))
+ Seq(Row(51, 100000004, "batch_1", 7843162, 400001276)))
checkAnswer(
sql("select city, count(id), sum(id), cast(avg(id) as integer), " +
@@ -382,54 +614,238 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
)
// non-filter
- val result = sql("select * from streaming.stream_table_filter_complex order by id").collect()
+ val result = sql("select * from streaming.stream_table_filter_complex order by id, name").collect()
assert(result != null)
assert(result.length == 55)
// check one row of streaming data
assert(result(0).isNullAt(0))
- assert(result(0).getString(1) == "name_6")
- assert(result(0).getStruct(4).getInt(1) == 6)
+ assert(result(0).getString(1) == "")
+ assert(result(0).getStruct(9).isNullAt(1))
// check one row of batch loading
assert(result(50).getInt(0) == 100000001)
assert(result(50).getString(1) == "batch_1")
- assert(result(50).getStruct(4).getInt(1) == 20)
+ assert(result(50).getStruct(9).getInt(1) == 20)
// filter
checkAnswer(
sql("select * from stream_table_filter_complex where id = 1"),
- Seq(Row(1, "name_1", "city_1", 10000.0, Row(wrap(Array("school_1", "school_11")), 1))))
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 1))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id > 49 and id < 100000002"),
+ Seq(Row(50, "name_50", "city_50", 500000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_50", "school_5050")), 50)),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 20))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id between 50 and 100000001"),
+ Seq(Row(50, "name_50", "city_50", 500000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_50", "school_5050")), 50)),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 20))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where name = 'name_3'"),
+ Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_3", "school_33")), 3))))
checkAnswer(
- sql("select * from stream_table_filter_complex where name = 'name_2'"),
- Seq(Row(2, "name_2", "", 20000.0, Row(wrap(Array("school_2", "school_22")), 2))))
+ sql("select * from stream_table_filter_complex where name like '%me_3%' and id < 30"),
+ Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_3", "school_33")), 3))))
checkAnswer(
- sql("select * from stream_table_filter_complex where file.age = 3"),
- Seq(Row(3, "name_3", "city_3", 30000.0, Row(wrap(Array("school_3", "school_33")), 3))))
+ sql("select * from stream_table_filter_complex where name >= 'name_3' and id < 4"),
+ Seq(Row(3, "name_3", "city_3", 30000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_3", "school_33")), 3))))
checkAnswer(
sql("select * from stream_table_filter_complex where city = 'city_1'"),
- Seq(Row(1, "name_1", "city_1", 10000.0, Row(wrap(Array("school_1", "school_11")), 1)),
- Row(100000001, "batch_1", "city_1", 0.1, Row(wrap(Array("school_1", "school_11")), 20))))
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 1)),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 20))))
checkAnswer(
- sql("select * from stream_table_filter_complex where id > 49 and id < 100000002"),
- Seq(Row(50, "name_50", "city_50", 500000.0, Row(wrap(Array("school_50", "school_5050")), 50)),
- Row(100000001, "batch_1", "city_1", 0.1, Row(wrap(Array("school_1", "school_11")), 20))))
+ sql("select * from stream_table_filter_complex where city like '%ty_1%' and ( id < 10 or id >= 100000001)"),
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 1)),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 20))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where city > 'city_09' and city < 'city_10'"),
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 1)),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 20))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where city between 'city_09' and 'city_1'"),
+ Seq(Row(1, "name_1", "city_1", 10000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 1)),
+ Row(100000001, "batch_1", "city_1", 0.1, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_1", "school_11")), 20))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where salary = 90000"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where salary > 80000 and salary <= 100000"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(10, "name_10", "city_10", 100000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_10", "school_1010")), 10))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where salary between 80001 and 90000"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where tax = 0.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where tax >= 0.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where tax < 0.05 and tax > 0.02 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where tax between 0.02 and 0.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where percent = 80.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where percent >= 80.04 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where percent < 80.05 and percent > 80.02 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where percent between 80.02 and 80.05 and id < 100"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where birthday between '1990-01-04' and '1990-01-05'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")), 50)),
+ Row(100000005, "batch_5", "city_5", 0.5, BigDecimal.valueOf(0.05), 80.05, Date.valueOf("1990-01-05"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Row(wrap(Array("school_5", "school_55")), 60))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where birthday = '1990-01-04'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")), 50))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where birthday > '1990-01-03' and birthday <= '1990-01-04'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")), 50))))
checkAnswer(
- sql("select * from stream_table_filter_complex where id is null"),
- Seq(Row(null, "name_6", "city_6", 60000.0, Row(wrap(Array("school_6", "school_66")), 6))))
+ sql("select * from stream_table_filter_complex where birthday between '1990-01-04' and '1990-01-05'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")), 50)),
+ Row(100000005, "batch_5", "city_5", 0.5, BigDecimal.valueOf(0.05), 80.05, Date.valueOf("1990-01-05"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Row(wrap(Array("school_5", "school_55")), 60))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where register = '2010-01-04 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")), 50))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where register > '2010-01-03 10:01:01' and register <= '2010-01-04 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")), 50))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where register between '2010-01-04 10:01:01' and '2010-01-05 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")),50)),
+ Row(100000005, "batch_5", "city_5", 0.5, BigDecimal.valueOf(0.05), 80.05, Date.valueOf("1990-01-05"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Row(wrap(Array("school_5", "school_55")), 60))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where updated = '2010-01-04 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")), 50))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where updated > '2010-01-03 10:01:01' and register <= '2010-01-04 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")), 50))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where updated between '2010-01-04 10:01:01' and '2010-01-05 10:01:01'"),
+ Seq(Row(9, "name_9", "city_9", 90000.0, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_9", "school_99")), 9)),
+ Row(100000004, "batch_4", "city_4", 0.4, BigDecimal.valueOf(0.04), 80.04, Date.valueOf("1990-01-04"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Timestamp.valueOf("2010-01-04 10:01:01.0"), Row(wrap(Array("school_4", "school_44")), 50)),
+ Row(100000005, "batch_5", "city_5", 0.5, BigDecimal.valueOf(0.05), 80.05, Date.valueOf("1990-01-05"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Timestamp.valueOf("2010-01-05 10:01:01.0"), Row(wrap(Array("school_5", "school_55")), 60))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id is null order by name"),
+ Seq(Row(null, "", "", null, null, null, null, null, null, Row(wrap(Array(null, null)), null)),
+ Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_6", "school_66")), 6))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where name = ''"),
+ Seq(Row(null, "", "", null, null, null, null, null, null, Row(wrap(Array(null, null)), null))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id is null and name <> ''"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_6", "school_66")), 6))))
checkAnswer(
sql("select * from stream_table_filter_complex where city = ''"),
- Seq(Row(2, "name_2", "", 20000.0, Row(wrap(Array("school_2", "school_22")), 2))))
+ Seq(Row(null, "", "", null, null, null, null, null, null, Row(wrap(Array(null, null)), null))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id is null and city <> ''"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_6", "school_66")), 6))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where salary is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null, Row(wrap(Array(null, null)), null))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id is null and salary is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_6", "school_66")), 6))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where tax is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null, Row(wrap(Array(null, null)), null))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id is null and tax is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_6", "school_66")), 6))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where percent is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null, Row(wrap(Array(null, null)), null))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id is null and salary is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_6", "school_66")), 6))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where birthday is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null, Row(wrap(Array(null, null)), null))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id is null and birthday is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_6", "school_66")), 6))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where register is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null, Row(wrap(Array(null, null)), null))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id is null and register is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_6", "school_66")), 6))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where updated is null"),
+ Seq(Row(null, "", "", null, null, null, null, null, null, Row(wrap(Array(null, null)), null))))
+
+ checkAnswer(
+ sql("select * from stream_table_filter_complex where id is null and updated is not null"),
+ Seq(Row(null, "name_6", "city_6", 60000.0, BigDecimal.valueOf(0.01), 80.01, Date.valueOf("1990-01-01"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Timestamp.valueOf("2010-01-01 10:01:01.0"), Row(wrap(Array("school_6", "school_66")), 6))))
// agg
checkAnswer(
sql("select count(*), max(id), min(name), cast(avg(file.age) as integer), sum(file.age) " +
"from stream_table_filter_complex where id >= 2 and id <= 100000004"),
- Seq(Row(52, 100000004, "batch_1", 27, 1408)))
+ Seq(Row(51, 100000004, "batch_1", 27, 1406)))
checkAnswer(
sql("select city, count(id), sum(id), cast(avg(file.age) as integer), " +
@@ -715,6 +1131,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
val clientSocket = serverSocket.accept()
val socketWriter = new PrintWriter(clientSocket.getOutputStream())
var index = 0
+ var timeRow = true
for (_ <- 1 to writeNums) {
// write 5 records per iteration
val stringBuilder = new StringBuilder()
@@ -723,22 +1140,32 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
if (badRecords) {
if (index == 2) {
// null value
- stringBuilder.append(index.toString + ",name_" + index
- + ",," + (10000.00 * index).toString +
- ",school_" + index + ":school_" + index + index + "$" + index)
+ stringBuilder.append(",,,,,,,,,")
} else if (index == 6) {
// illegal number
stringBuilder.append(index.toString + "abc,name_" + index
- + ",city_" + index + "," + (10000.00 * index).toString +
+ + ",city_" + index + "," + (10000.00 * index).toString + ",0.01,80.01" +
+ ",1990-01-01,2010-01-01 10:01:01,2010-01-01 10:01:01" +
",school_" + index + ":school_" + index + index + "$" + index)
} else {
- stringBuilder.append(index.toString + ",name_" + index
- + ",city_" + index + "," + (10000.00 * index).toString +
- ",school_" + index + ":school_" + index + index + "$" + index)
+
+ if (index == 9 && timeRow) {
+ timeRow = false
+ stringBuilder.append(index.toString + ",name_" + index
+ + ",city_" + index + "," + (10000.00 * index).toString + ",0.04,80.04" +
+ ",1990-01-04,2010-01-04 10:01:01,2010-01-04 10:01:01" +
+ ",school_" + index + ":school_" + index + index + "$" + index)
+ } else {
+ stringBuilder.append(index.toString + ",name_" + index
+ + ",city_" + index + "," + (10000.00 * index).toString + ",0.01,80.01" +
+ ",1990-01-01,2010-01-01 10:01:01,2010-01-01 10:01:01" +
+ ",school_" + index + ":school_" + index + index + "$" + index)
+ }
}
} else {
stringBuilder.append(index.toString + ",name_" + index
- + ",city_" + index + "," + (10000.00 * index).toString +
+ + ",city_" + index + "," + (10000.00 * index).toString + ",0.01,80.01" +
+ ",1990-01-01,2010-01-01 10:01:01,2010-01-01 10:01:01" +
",school_" + index + ":school_" + index + index + "$" + index)
}
stringBuilder.append("\n")
@@ -781,6 +1208,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
.option("dbName", tableIdentifier.database.get)
.option("tableName", tableIdentifier.table)
.option(CarbonCommonConstants.HANDOFF_SIZE, handoffSize)
+ .option("timestampformat", CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
.option(CarbonCommonConstants.ENABLE_AUTO_HANDOFF, autoHandoff)
.start()
qry.awaitTermination()
@@ -857,9 +1285,15 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
"name_" + id,
"city_" + id,
10000.00 * id,
+ BigDecimal.valueOf(0.01),
+ 80.01,
+ "1990-01-01",
+ "2010-01-01 10:01:01",
+ "2010-01-01 10:01:01",
"school_" + id + ":school_" + id + id + "$" + id)
}
- val csvDataDF = spark.createDataFrame(csvRDD).toDF("id", "name", "city", "salary", "file")
+ val csvDataDF = spark.createDataFrame(csvRDD).toDF(
+ "id", "name", "city", "salary", "tax", "percent", "birthday", "register", "updated", "file")
csvDataDF.write
.option("header", "false")
@@ -875,12 +1309,6 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
tableIdentifier: TableIdentifier): Thread = {
new Thread() {
override def run(): Unit = {
- val inputSchema = new StructType()
- .add("id", "integer")
- .add("name", "string")
- .add("city", "string")
- .add("salary", "float")
- .add("file", "string")
var qry: StreamingQuery = null
try {
val readSocketDF = spark.readStream.text(csvDataDir)
@@ -892,6 +1320,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
.option("checkpointLocation", tablePath.getStreamingCheckpointDir)
.option("dbName", tableIdentifier.database.get)
.option("tableName", tableIdentifier.table)
+ .option("timestampformat", CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
.start()
qry.awaitTermination()
@@ -914,11 +1343,16 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
| id INT,
| name STRING,
| city STRING,
- | salary FLOAT
+ | salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP
| )
| STORED BY 'carbondata'
| TBLPROPERTIES(${if (streaming) "'streaming'='true', " else "" }
- | 'sort_columns'='name', 'dictionary_include'='city')
+ | 'sort_columns'='name', 'dictionary_include'='city,register')
| """.stripMargin)
if (withBatchLoad) {
@@ -938,11 +1372,16 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
| name STRING,
| city STRING,
| salary FLOAT,
+ | tax DECIMAL(8,2),
+ | percent double,
+ | birthday DATE,
+ | register TIMESTAMP,
+ | updated TIMESTAMP,
| file struct<school:array<string>, age:int>
| )
| STORED BY 'carbondata'
| TBLPROPERTIES(${if (streaming) "'streaming'='true', " else "" }
- | 'sort_columns'='name', 'dictionary_include'='city')
+ | 'sort_columns'='name', 'dictionary_include'='id,name,salary,tax,percent,updated')
| """.stripMargin)
if (withBatchLoad) {
[15/16] carbondata git commit: [CARBONDATA-2187][PARTITION] Partition
restructure for new folder structure and supporting partition location
feature
Posted by gv...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
deleted file mode 100644
index a0ce24a..0000000
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ /dev/null
@@ -1,484 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.metadata;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Serializable;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
-import org.apache.carbondata.core.fileoperations.FileWriteOperation;
-import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
-import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
-import org.apache.carbondata.core.statusmanager.SegmentStatus;
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataFileFooterConverter;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter;
-
-import com.google.gson.Gson;
-
-/**
- * Provide read and write support for partition mapping file in each segment
- */
-public class PartitionMapFileStore {
-
- private Map<String, List<String>> partitionMap = new HashMap<>();
-
- private boolean partionedSegment = false;
- /**
- * Write partitionmapp file to the segment folder with indexfilename and corresponding partitions.
- *
- * @param segmentPath
- * @param taskNo
- * @param partionNames
- * @throws IOException
- */
- public void writePartitionMapFile(String segmentPath, final String taskNo,
- List<String> partionNames) throws IOException {
- CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
- // write partition info to new file.
- if (carbonFile.exists() && partionNames.size() > 0) {
- CarbonFile[] carbonFiles = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().startsWith(taskNo) && file.getName()
- .endsWith(CarbonTablePath.INDEX_FILE_EXT);
- }
- });
- if (carbonFiles != null && carbonFiles.length > 0) {
- PartitionMapper partitionMapper = new PartitionMapper();
- Map<String, List<String>> partitionMap = new HashMap<>();
- partitionMap.put(carbonFiles[0].getName(), partionNames);
- partitionMapper.setPartitionMap(partitionMap);
- String path = segmentPath + "/" + taskNo + CarbonTablePath.PARTITION_MAP_EXT;
- writePartitionFile(partitionMapper, path);
- }
- }
- }
-
- private void writePartitionFile(PartitionMapper partitionMapper, String path) throws IOException {
- AtomicFileOperations fileWrite =
- new AtomicFileOperationsImpl(path, FileFactory.getFileType(path));
- BufferedWriter brWriter = null;
- DataOutputStream dataOutputStream = null;
- Gson gsonObjectToWrite = new Gson();
- try {
- dataOutputStream = fileWrite.openForWrite(FileWriteOperation.OVERWRITE);
- brWriter = new BufferedWriter(new OutputStreamWriter(dataOutputStream,
- Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
-
- String metadataInstance = gsonObjectToWrite.toJson(partitionMapper);
- brWriter.write(metadataInstance);
- } finally {
- if (null != brWriter) {
- brWriter.flush();
- }
- CarbonUtil.closeStreams(brWriter);
- fileWrite.close();
- }
- }
-
- /**
- * Merge all partition files in a segment to single file.
- *
- * @param segmentPath
- * @throws IOException
- */
- public void mergePartitionMapFiles(String segmentPath, String mergeFileName) throws IOException {
- CarbonFile[] partitionFiles = getPartitionFiles(segmentPath);
- if (partitionFiles != null && partitionFiles.length > 0) {
- PartitionMapper partitionMapper = null;
- for (CarbonFile file : partitionFiles) {
- PartitionMapper localMapper = readPartitionMap(file.getAbsolutePath());
- if (partitionMapper == null && localMapper != null) {
- partitionMapper = localMapper;
- }
- if (localMapper != null) {
- partitionMapper = partitionMapper.merge(localMapper);
- }
- }
- if (partitionMapper != null) {
- String path = segmentPath + "/" + mergeFileName + CarbonTablePath.PARTITION_MAP_EXT;
- writePartitionFile(partitionMapper, path);
- for (CarbonFile file : partitionFiles) {
- if (!FileFactory.deleteAllCarbonFilesOfDir(file)) {
- throw new IOException("Old partition map files cannot be deleted");
- }
- }
- }
- }
- }
-
- private String getPartitionFilePath(String segmentPath) {
- CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
- if (carbonFile.exists()) {
- CarbonFile[] partitionFiles = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().endsWith(CarbonTablePath.PARTITION_MAP_EXT);
- }
- });
- if (partitionFiles != null && partitionFiles.length > 0) {
- partionedSegment = true;
- int i = 0;
- // Get the latest partition map file based on the timestamp of that file.
- long[] partitionTimestamps = new long[partitionFiles.length];
- for (CarbonFile file : partitionFiles) {
- partitionTimestamps[i++] = Long.parseLong(file.getName()
- .substring(0, file.getName().length() - CarbonTablePath.PARTITION_MAP_EXT.length()));
- }
- Arrays.sort(partitionTimestamps);
- return segmentPath + "/" + partitionTimestamps[partitionTimestamps.length - 1]
- + CarbonTablePath.PARTITION_MAP_EXT;
- }
- }
- return null;
- }
-
- private String getPartitionFilePath(CarbonFile[] carbonFiles, String segmentPath) {
-
- List<CarbonFile> partitionFiles = new ArrayList<>();
- for (CarbonFile file : carbonFiles) {
- if (file.getName().endsWith(CarbonTablePath.PARTITION_MAP_EXT)) {
- partitionFiles.add(file);
- }
- }
- if (partitionFiles.size() > 0) {
- partionedSegment = true;
- int i = 0;
- // Get the latest partition map file based on the timestamp of that file.
- long[] partitionTimestamps = new long[partitionFiles.size()];
- for (CarbonFile file : partitionFiles) {
- partitionTimestamps[i++] = Long.parseLong(file.getName()
- .substring(0, file.getName().length() - CarbonTablePath.PARTITION_MAP_EXT.length()));
- }
- Arrays.sort(partitionTimestamps);
- return segmentPath + "/" + partitionTimestamps[partitionTimestamps.length - 1]
- + CarbonTablePath.PARTITION_MAP_EXT;
- }
- return null;
- }
-
- private CarbonFile[] getPartitionFiles(String segmentPath) {
- CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
- if (carbonFile.exists()) {
- return carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().endsWith(CarbonTablePath.PARTITION_MAP_EXT);
- }
- });
- }
- return null;
- }
-
- /**
- * This method reads the partition file
- *
- * @param partitionMapPath
- * @return
- */
- private PartitionMapper readPartitionMap(String partitionMapPath) throws IOException {
- Gson gsonObjectToRead = new Gson();
- DataInputStream dataInputStream = null;
- BufferedReader buffReader = null;
- InputStreamReader inStream = null;
- PartitionMapper partitionMapper;
- AtomicFileOperations fileOperation =
- new AtomicFileOperationsImpl(partitionMapPath, FileFactory.getFileType(partitionMapPath));
-
- try {
- if (!FileFactory.isFileExist(partitionMapPath, FileFactory.getFileType(partitionMapPath))) {
- return null;
- }
- dataInputStream = fileOperation.openForRead();
- inStream = new InputStreamReader(dataInputStream,
- Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
- buffReader = new BufferedReader(inStream);
- partitionMapper = gsonObjectToRead.fromJson(buffReader, PartitionMapper.class);
- } finally {
- if (inStream != null) {
- CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
- }
- }
-
- return partitionMapper;
- }
-
- /**
- * Reads all partitions which existed inside the passed segment path
- * @param segmentPath
- */
- public void readAllPartitionsOfSegment(String segmentPath) throws IOException {
- String partitionFilePath = getPartitionFilePath(segmentPath);
- if (partitionFilePath != null) {
- partionedSegment = true;
- PartitionMapper partitionMapper = readPartitionMap(partitionFilePath);
- partitionMap.putAll(partitionMapper.getPartitionMap());
- }
- }
-
- /**
- * Reads all partitions which existed inside the passed segment path
- * @param carbonFiles
- */
- public void readAllPartitionsOfSegment(CarbonFile[] carbonFiles, String segmentPath)
- throws IOException {
- String partitionFilePath = getPartitionFilePath(carbonFiles, segmentPath);
- if (partitionFilePath != null) {
- partionedSegment = true;
- PartitionMapper partitionMapper = readPartitionMap(partitionFilePath);
- partitionMap.putAll(partitionMapper.getPartitionMap());
- }
- }
-
- public boolean isPartionedSegment() {
- return partionedSegment;
- }
-
- /**
- * Drops the partitions from the partition mapper file of the segment and writes to a new file.
- * @param segmentPath
- * @param partitionsToDrop
- * @param uniqueId
- * @param partialMatch If it is true then even the partial partition spec matches also can be
- * dropped
- * @throws IOException
- */
- public void dropPartitions(String segmentPath, List<List<String>> partitionsToDrop,
- String uniqueId, boolean partialMatch) throws IOException {
- readAllPartitionsOfSegment(segmentPath);
- List<String> indexesToDrop = new ArrayList<>();
- for (Map.Entry<String, List<String>> entry: partitionMap.entrySet()) {
- for (List<String> partitions: partitionsToDrop) {
- if (partialMatch) {
- if (entry.getValue().containsAll(partitions)) {
- indexesToDrop.add(entry.getKey());
- }
- } else {
- if (partitions.containsAll(entry.getValue())) {
- indexesToDrop.add(entry.getKey());
- }
- }
- }
- }
- if (indexesToDrop.size() > 0) {
- // Remove the indexes from partition map
- for (String indexToDrop : indexesToDrop) {
- partitionMap.remove(indexToDrop);
- }
- PartitionMapper mapper = new PartitionMapper();
- mapper.setPartitionMap(partitionMap);
- String path = segmentPath + "/" + uniqueId + CarbonTablePath.PARTITION_MAP_EXT + ".tmp";
- writePartitionFile(mapper, path);
- }
- }
-
- /**
- * It deletes the old partition mapper files in case of success. And in case of failure it removes
- * the old new file.
- * @param segmentPath
- * @param uniqueId
- * @param success
- */
- public void commitPartitions(String segmentPath, final String uniqueId, boolean success,
- String tablePath, List<String> partitionsToDrop) {
- CarbonFile carbonFile = FileFactory
- .getCarbonFile(segmentPath + "/" + uniqueId + CarbonTablePath.PARTITION_MAP_EXT + ".tmp");
- CarbonFile carbonPartFile = FileFactory
- .getCarbonFile(tablePath + "/" + partitionsToDrop.get(0));
- // write partition info to new file.
- if (carbonFile.exists()) {
- if (success) {
- carbonFile.renameForce(segmentPath + "/" + uniqueId + CarbonTablePath.PARTITION_MAP_EXT);
- } else {
- carbonFile.delete();
- }
- }
- //Remove the partition directory from table path
- carbonPartFile.delete();
- }
-
- /**
- * Clean up invalid data after drop partition in all segments of table
- * @param table
- * @param currentPartitions Current partitions of table
- * @param forceDelete Whether it should be deleted force or check the time for an hour creation
- * to delete data.
- * @throws IOException
- */
- public void cleanSegments(
- CarbonTable table,
- List<String> currentPartitions,
- boolean forceDelete) throws IOException {
- SegmentStatusManager ssm = new SegmentStatusManager(table.getAbsoluteTableIdentifier());
-
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(table.getAbsoluteTableIdentifier().getTablePath(),
- table.getAbsoluteTableIdentifier().getCarbonTableIdentifier());
-
- LoadMetadataDetails[] details = ssm.readLoadMetadata(table.getMetaDataFilepath());
- // scan through each segment.
- List<String> segmentsNeedToBeDeleted = new ArrayList<>();
- for (LoadMetadataDetails segment : details) {
-
- // if this segment is valid then only we will go for deletion of related
- // dropped partition files. if the segment is mark for delete or compacted then any way
- // it will get deleted.
-
- if (segment.getSegmentStatus() == SegmentStatus.SUCCESS
- || segment.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
- List<String> toBeDeletedIndexFiles = new ArrayList<>();
- List<String> toBeDeletedDataFiles = new ArrayList<>();
- // take the list of files from this segment.
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment.getLoadName());
- String partitionFilePath = getPartitionFilePath(segmentPath);
- if (partitionFilePath != null) {
- PartitionMapper partitionMapper = readPartitionMap(partitionFilePath);
- if (partitionMapper.partitionMap.size() == 0) {
- // There is no partition information, it means all partitions are dropped.
- // So segment need to be marked as delete.
- segmentsNeedToBeDeleted.add(segment.getLoadName());
- continue;
- }
- DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
- SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
- indexFileStore.readAllIIndexOfSegment(segmentPath);
- Set<String> indexFilesFromSegment = indexFileStore.getCarbonIndexMap().keySet();
- for (String indexFile : indexFilesFromSegment) {
- // Check the partition information in the partiton mapper
- List<String> indexPartitions = partitionMapper.partitionMap.get(indexFile);
- if (indexPartitions == null || !currentPartitions.containsAll(indexPartitions)) {
- Long fileTimestamp = CarbonUpdateUtil.getTimeStampAsLong(indexFile
- .substring(indexFile.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
- indexFile.length() - CarbonTablePath.INDEX_FILE_EXT.length()));
- if (CarbonUpdateUtil.isMaxQueryTimeoutExceeded(fileTimestamp) || forceDelete) {
- toBeDeletedIndexFiles.add(indexFile);
- // Add the corresponding carbondata files to the delete list.
- byte[] fileData = indexFileStore.getFileData(indexFile);
- List<DataFileFooter> indexInfo =
- fileFooterConverter.getIndexInfo(segmentPath + "/" + indexFile, fileData);
- for (DataFileFooter footer : indexInfo) {
- toBeDeletedDataFiles.add(footer.getBlockInfo().getTableBlockInfo().getFilePath());
- }
- }
- }
- }
-
- if (toBeDeletedIndexFiles.size() > 0) {
- indexFilesFromSegment.removeAll(toBeDeletedIndexFiles);
- new CarbonIndexFileMergeWriter().mergeCarbonIndexFilesOfSegment(segmentPath,
- new ArrayList<String>(indexFilesFromSegment));
- for (String dataFile : toBeDeletedDataFiles) {
- FileFactory.deleteFile(dataFile, FileFactory.getFileType(dataFile));
- }
- }
- CarbonFile[] partitionFiles = getPartitionFiles(segmentPath);
- CarbonFile currentPartitionFile = FileFactory.getCarbonFile(partitionFilePath);
- if (partitionFiles != null) {
- // Delete all old partition files
- for (CarbonFile partitionFile : partitionFiles) {
- if (!currentPartitionFile.getName().equalsIgnoreCase(partitionFile.getName())) {
- long fileTimeStamp = Long.parseLong(partitionFile.getName().substring(0,
- partitionFile.getName().length() - CarbonTablePath.PARTITION_MAP_EXT.length()));
- if (CarbonUpdateUtil.isMaxQueryTimeoutExceeded(fileTimeStamp) || forceDelete) {
- partitionFile.delete();
- }
- }
- }
- }
- partitionMapper = readPartitionMap(partitionFilePath);
- if (partitionMapper != null) {
- // delete partition map if there is no partition files exist
- if (partitionMapper.partitionMap.size() == 0) {
- currentPartitionFile.delete();
- }
- }
- }
- }
- }
- // If any segments that are required to delete
- if (segmentsNeedToBeDeleted.size() > 0) {
- try {
- // Mark the segments as delete.
- SegmentStatusManager.updateDeletionStatus(
- table.getAbsoluteTableIdentifier(),
- segmentsNeedToBeDeleted,
- table.getMetaDataFilepath());
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
- }
-
- public List<String> getPartitions(String indexFileName) {
- return partitionMap.get(indexFileName);
- }
-
- public Map<String, List<String>> getPartitionMap() {
- return partitionMap;
- }
-
- public static class PartitionMapper implements Serializable {
-
- private static final long serialVersionUID = 3582245668420401089L;
-
- private Map<String, List<String>> partitionMap;
-
- public PartitionMapper merge(PartitionMapper mapper) {
- if (this == mapper) {
- return this;
- }
- if (partitionMap != null && mapper.partitionMap != null) {
- partitionMap.putAll(mapper.partitionMap);
- }
- if (partitionMap == null) {
- partitionMap = mapper.partitionMap;
- }
- return this;
- }
-
- public Map<String, List<String>> getPartitionMap() {
- return partitionMap;
- }
-
- public void setPartitionMap(Map<String, List<String>> partitionMap) {
- this.partitionMap = partitionMap;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
new file mode 100644
index 0000000..b5f5a25
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -0,0 +1,744 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import com.google.gson.Gson;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Provide read and write support for segment file associated with each segment
+ */
+public class SegmentFileStore {
+
+ private SegmentFile segmentFile;
+
+ /**
+ * Here key folder path and values are index files in it.
+ */
+ private Map<String, List<String>> indexFilesMap;
+
+ private String tablePath;
+
+ public SegmentFileStore(String tablePath, String segmentFileName) throws IOException {
+ this.tablePath = tablePath;
+ this.segmentFile = readSegment(tablePath, segmentFileName);
+ }
+
+ /**
+ * Write segment information to the segment folder with indexfilename and
+ * corresponding partitions.
+ */
+ public static void writeSegmentFile(String tablePath, final String taskNo, String location,
+ String timeStamp, List<String> partionNames) throws IOException {
+ String tempFolderLoc = timeStamp + ".tmp";
+ String writePath = CarbonTablePath.getSegmentFilesLocation(tablePath) + "/" + tempFolderLoc;
+ CarbonFile carbonFile = FileFactory.getCarbonFile(writePath);
+ if (!carbonFile.exists()) {
+ carbonFile.mkdirs(writePath, FileFactory.getFileType(writePath));
+ }
+ CarbonFile tempFolder =
+ FileFactory.getCarbonFile(location + CarbonCommonConstants.FILE_SEPARATOR + tempFolderLoc);
+
+ if (tempFolder.exists() && partionNames.size() > 0) {
+ CarbonFile[] carbonFiles = tempFolder.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().startsWith(taskNo) && file.getName()
+ .endsWith(CarbonTablePath.INDEX_FILE_EXT);
+ }
+ });
+ if (carbonFiles != null && carbonFiles.length > 0) {
+ boolean isRelative = false;
+ if (location.startsWith(tablePath)) {
+ location = location.substring(tablePath.length(), location.length());
+ isRelative = true;
+ }
+ SegmentFile segmentFile = new SegmentFile();
+ Map<String, FolderDetails> locationMap = new HashMap<>();
+ FolderDetails folderDetails = new FolderDetails();
+ folderDetails.setRelative(isRelative);
+ folderDetails.setPartitions(partionNames);
+ folderDetails.setStatus(SegmentStatus.SUCCESS.getMessage());
+ for (CarbonFile file : carbonFiles) {
+ folderDetails.getFiles().add(file.getName());
+ }
+ locationMap.put(location, folderDetails);
+ segmentFile.setLocationMap(locationMap);
+ String path = writePath + "/" + taskNo + CarbonTablePath.SEGMENT_EXT;
+ // write segment info to new file.
+ writeSegmentFile(segmentFile, path);
+ }
+ }
+ }
+
+ /**
+ * Writes the segment file in json format
+ * @param segmentFile
+ * @param path
+ * @throws IOException
+ */
+ public static void writeSegmentFile(SegmentFile segmentFile, String path) throws IOException {
+ AtomicFileOperations fileWrite =
+ new AtomicFileOperationsImpl(path, FileFactory.getFileType(path));
+ BufferedWriter brWriter = null;
+ DataOutputStream dataOutputStream = null;
+ Gson gsonObjectToWrite = new Gson();
+ try {
+ dataOutputStream = fileWrite.openForWrite(FileWriteOperation.OVERWRITE);
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutputStream,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+
+ String metadataInstance = gsonObjectToWrite.toJson(segmentFile);
+ brWriter.write(metadataInstance);
+ brWriter.flush();
+ } finally {
+ CarbonUtil.closeStreams(brWriter);
+ fileWrite.close();
+ }
+ }
+
+ /**
+ * Merge all segment files in a segment to single file.
+ *
+ * @throws IOException
+ */
+ public static SegmentFile mergeSegmentFiles(String readPath, String mergeFileName,
+ String writePath)
+ throws IOException {
+ CarbonFile[] segmentFiles = getSegmentFiles(readPath);
+ if (segmentFiles != null && segmentFiles.length > 0) {
+ SegmentFile segmentFile = null;
+ for (CarbonFile file : segmentFiles) {
+ SegmentFile localSegmentFile = readSegmentFile(file.getAbsolutePath());
+ if (segmentFile == null && localSegmentFile != null) {
+ segmentFile = localSegmentFile;
+ }
+ if (localSegmentFile != null) {
+ segmentFile = segmentFile.merge(localSegmentFile);
+ }
+ }
+ if (segmentFile != null) {
+ String path = writePath + "/" + mergeFileName + CarbonTablePath.SEGMENT_EXT;
+ writeSegmentFile(segmentFile, path);
+ FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(readPath));
+ }
+ return segmentFile;
+ }
+ return null;
+ }
+
+ private static CarbonFile[] getSegmentFiles(String segmentPath) {
+ CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
+ if (carbonFile.exists()) {
+ return carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(CarbonTablePath.SEGMENT_EXT);
+ }
+ });
+ }
+ return null;
+ }
+
+ /**
+ * It provides segment file only for the partitions which has physical index files.
+ *
+ * @param partitionSpecs
+ */
+ public static SegmentFile getSegmentFileForPhysicalDataPartitions(String tablePath,
+ List<PartitionSpec> partitionSpecs) {
+ SegmentFile segmentFile = null;
+ for (PartitionSpec spec : partitionSpecs) {
+ String location = spec.getLocation().toString();
+ CarbonFile carbonFile = FileFactory.getCarbonFile(location);
+
+ CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return CarbonTablePath.isCarbonIndexFile(file.getAbsolutePath());
+ }
+ });
+ if (listFiles != null && listFiles.length > 0) {
+ boolean isRelative = false;
+ if (location.startsWith(tablePath)) {
+ location = location.substring(tablePath.length(), location.length());
+ isRelative = true;
+ }
+ SegmentFile localSegmentFile = new SegmentFile();
+ Map<String, FolderDetails> locationMap = new HashMap<>();
+ FolderDetails folderDetails = new FolderDetails();
+ folderDetails.setRelative(isRelative);
+ folderDetails.setPartitions(spec.getPartitions());
+ folderDetails.setStatus(SegmentStatus.SUCCESS.getMessage());
+ for (CarbonFile file : listFiles) {
+ if (file.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
+ folderDetails.setMergeFileName(file.getName());
+ } else {
+ folderDetails.getFiles().add(file.getName());
+ }
+ }
+ locationMap.put(location, folderDetails);
+ localSegmentFile.setLocationMap(locationMap);
+ if (segmentFile == null) {
+ segmentFile = localSegmentFile;
+ } else {
+ segmentFile = segmentFile.merge(localSegmentFile);
+ }
+ }
+ }
+ return segmentFile;
+ }
+
+ /**
+ * This method reads the segment file which is written in json format
+ *
+ * @param segmentFilePath
+ * @return
+ */
+ private static SegmentFile readSegmentFile(String segmentFilePath) throws IOException {
+ Gson gsonObjectToRead = new Gson();
+ DataInputStream dataInputStream = null;
+ BufferedReader buffReader = null;
+ InputStreamReader inStream = null;
+ SegmentFile segmentFile;
+ AtomicFileOperations fileOperation =
+ new AtomicFileOperationsImpl(segmentFilePath, FileFactory.getFileType(segmentFilePath));
+
+ try {
+ if (!FileFactory.isFileExist(segmentFilePath, FileFactory.getFileType(segmentFilePath))) {
+ return null;
+ }
+ dataInputStream = fileOperation.openForRead();
+ inStream = new InputStreamReader(dataInputStream,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ buffReader = new BufferedReader(inStream);
+ segmentFile = gsonObjectToRead.fromJson(buffReader, SegmentFile.class);
+ } finally {
+ if (inStream != null) {
+ CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
+ }
+ }
+
+ return segmentFile;
+ }
+
+ /**
+ * Reads segment file.
+ */
+ private SegmentFile readSegment(String tablePath, String segmentFileName) throws IOException {
+ String segmentFilePath =
+ CarbonTablePath.getSegmentFilesLocation(tablePath) + CarbonCommonConstants.FILE_SEPARATOR
+ + segmentFileName;
+ return readSegmentFile(segmentFilePath);
+ }
+
+ public String getTablePath() {
+ return tablePath;
+ }
+
+ /**
+ * Gets all the index files and related carbondata files from this segment. First user needs to
+ * call @readIndexFiles method before calling it.
+ * @return
+ */
+ public Map<String, List<String>> getIndexFilesMap() {
+ return indexFilesMap;
+ }
+
+ /**
+ * Reads all index files which are located in this segment. First user needs to call
+ * @readSegment method before calling it.
+ * @throws IOException
+ */
+ public void readIndexFiles() throws IOException {
+ readIndexFiles(SegmentStatus.SUCCESS, false);
+ }
+
+ /**
+ * Reads all index files as per the status of the file. In case of @ignoreStatus is true it just
+ * reads all index files
+ * @param status
+ * @param ignoreStatus
+ * @throws IOException
+ */
+ private void readIndexFiles(SegmentStatus status, boolean ignoreStatus) throws IOException {
+ if (indexFilesMap != null) {
+ return;
+ }
+ SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
+ indexFilesMap = new HashMap<>();
+ indexFileStore.readAllIIndexOfSegment(this, status, ignoreStatus);
+ Map<String, byte[]> carbonIndexMap = indexFileStore.getCarbonIndexMapWithFullPath();
+ DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
+ for (Map.Entry<String, byte[]> entry : carbonIndexMap.entrySet()) {
+ List<DataFileFooter> indexInfo =
+ fileFooterConverter.getIndexInfo(entry.getKey(), entry.getValue());
+ List<String> blocks = new ArrayList<>();
+ for (DataFileFooter footer : indexInfo) {
+ blocks.add(footer.getBlockInfo().getTableBlockInfo().getFilePath());
+ }
+ indexFilesMap.put(entry.getKey(), blocks);
+ }
+ }
+
+ /**
+ * Gets all index files from this segment
+ * @return
+ */
+ public Map<String, String> getIndexFiles() {
+ Map<String, String> indexFiles = new HashMap<>();
+ if (segmentFile != null) {
+ for (Map.Entry<String, FolderDetails> entry : getLocationMap().entrySet()) {
+ String location = entry.getKey();
+ if (entry.getValue().isRelative) {
+ location = tablePath + CarbonCommonConstants.FILE_SEPARATOR + location;
+ }
+ if (entry.getValue().status.equals(SegmentStatus.SUCCESS.getMessage())) {
+ for (String indexFile : entry.getValue().getFiles()) {
+ indexFiles.put(location + CarbonCommonConstants.FILE_SEPARATOR + indexFile,
+ entry.getValue().mergeFileName);
+ }
+ }
+ }
+ }
+ return indexFiles;
+ }
+
+ /**
+ * Drops the partition related files from the segment file of the segment and writes
+ * to a new file. First iterator over segment file and check the path it needs to be dropped.
+ * And update the status with delete if it found.
+ *
+ * @param uniqueId
+ * @throws IOException
+ */
+ public void dropPartitions(Segment segment, List<PartitionSpec> partitionSpecs,
+ String uniqueId, List<String> toBeDeletedSegments, List<String> toBeUpdatedSegments)
+ throws IOException {
+ readSegment(tablePath, segment.getSegmentFileName());
+ boolean updateSegment = false;
+ for (Map.Entry<String, FolderDetails> entry : segmentFile.getLocationMap().entrySet()) {
+ String location = entry.getKey();
+ if (entry.getValue().isRelative) {
+ location = tablePath + CarbonCommonConstants.FILE_SEPARATOR + location;
+ }
+ Path path = new Path(location);
+ // Update the status to delete if path equals
+ for (PartitionSpec spec : partitionSpecs) {
+ if (path.equals(spec.getLocation())) {
+ entry.getValue().setStatus(SegmentStatus.MARKED_FOR_DELETE.getMessage());
+ updateSegment = true;
+ break;
+ }
+ }
+ }
+ String writePath = CarbonTablePath.getSegmentFilesLocation(tablePath);
+ writePath =
+ writePath + CarbonCommonConstants.FILE_SEPARATOR + segment.getSegmentNo() + "_" + uniqueId
+ + CarbonTablePath.SEGMENT_EXT;
+ writeSegmentFile(segmentFile, writePath);
+ // Check whether we can completly remove the segment.
+ boolean deleteSegment = true;
+ for (Map.Entry<String, FolderDetails> entry : segmentFile.getLocationMap().entrySet()) {
+ if (entry.getValue().getStatus().equals(SegmentStatus.SUCCESS.getMessage())) {
+ deleteSegment = false;
+ break;
+ }
+ }
+ if (deleteSegment) {
+ toBeDeletedSegments.add(segment.getSegmentNo());
+ }
+ if (updateSegment) {
+ toBeUpdatedSegments.add(segment.getSegmentNo());
+ }
+ }
+
+ /**
+ * Update the table status file with the dropped partitions information
+ *
+ * @param carbonTable
+ * @param uniqueId
+ * @param toBeUpdatedSegments
+ * @param toBeDeleteSegments
+ * @throws IOException
+ */
+ public static void commitDropPartitions(CarbonTable carbonTable, String uniqueId,
+ List<String> toBeUpdatedSegments, List<String> toBeDeleteSegments) throws IOException {
+ if (toBeDeleteSegments.size() > 0 || toBeUpdatedSegments.size() > 0) {
+ Set<Segment> segmentSet = new HashSet<>(
+ new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier())
+ .getValidAndInvalidSegments().getValidSegments());
+ CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, uniqueId, true,
+ Segment.toSegmentList(toBeDeleteSegments), Segment.toSegmentList(toBeUpdatedSegments));
+ }
+ }
+
+ /**
+ * Clean up invalid data after drop partition in all segments of table
+ *
+ * @param table
+ * @param forceDelete Whether it should be deleted force or check the time for an hour creation
+ * to delete data.
+ * @throws IOException
+ */
+ public static void cleanSegments(CarbonTable table, List<PartitionSpec> partitionSpecs,
+ boolean forceDelete) throws IOException {
+
+ LoadMetadataDetails[] details =
+ SegmentStatusManager.readLoadMetadata(table.getMetaDataFilepath());
+ // scan through each segment.
+ for (LoadMetadataDetails segment : details) {
+ // if this segment is valid then only we will go for deletion of related
+ // dropped partition files. if the segment is mark for delete or compacted then any way
+ // it will get deleted.
+
+ if ((segment.getSegmentStatus() == SegmentStatus.SUCCESS
+ || segment.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS)
+ && segment.getSegmentFile() != null) {
+ List<String> toBeDeletedIndexFiles = new ArrayList<>();
+ List<String> toBeDeletedDataFiles = new ArrayList<>();
+ // take the list of files from this segment.
+ SegmentFileStore fileStore =
+ new SegmentFileStore(table.getTablePath(), segment.getSegmentFile());
+ fileStore.readIndexFiles(SegmentStatus.MARKED_FOR_DELETE, false);
+ if (forceDelete) {
+ deletePhysicalPartition(partitionSpecs, fileStore.getIndexFilesMap());
+ }
+ for (Map.Entry<String, List<String>> entry : fileStore.indexFilesMap.entrySet()) {
+ String indexFile = entry.getKey();
+ // Check the partition information in the partiton mapper
+ Long fileTimestamp = CarbonUpdateUtil.getTimeStampAsLong(indexFile
+ .substring(indexFile.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
+ indexFile.length() - CarbonTablePath.INDEX_FILE_EXT.length()));
+ if (CarbonUpdateUtil.isMaxQueryTimeoutExceeded(fileTimestamp) || forceDelete) {
+ toBeDeletedIndexFiles.add(indexFile);
+ // Add the corresponding carbondata files to the delete list.
+ toBeDeletedDataFiles.addAll(entry.getValue());
+ }
+ }
+ if (toBeDeletedIndexFiles.size() > 0) {
+ for (String dataFile : toBeDeletedIndexFiles) {
+ FileFactory.deleteFile(dataFile, FileFactory.getFileType(dataFile));
+ }
+ for (String dataFile : toBeDeletedDataFiles) {
+ FileFactory.deleteFile(dataFile, FileFactory.getFileType(dataFile));
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Deletes the segment file and its physical files like partition folders from disk
+ * @param tablePath
+ * @param segmentFile
+ * @param partitionSpecs
+ * @throws IOException
+ */
+ public static void deleteSegment(String tablePath, String segmentFile,
+ List<PartitionSpec> partitionSpecs) throws IOException {
+ SegmentFileStore fileStore = new SegmentFileStore(tablePath, segmentFile);
+ fileStore.readIndexFiles(SegmentStatus.SUCCESS, true);
+ Map<String, List<String>> indexFilesMap = fileStore.getIndexFilesMap();
+ for (Map.Entry<String, List<String>> entry : indexFilesMap.entrySet()) {
+ FileFactory.deleteFile(entry.getKey(), FileFactory.getFileType(entry.getKey()));
+ for (String file : entry.getValue()) {
+ FileFactory.deleteFile(file, FileFactory.getFileType(file));
+ }
+ }
+ deletePhysicalPartition(partitionSpecs, indexFilesMap);
+ String segmentFilePath =
+ CarbonTablePath.getSegmentFilesLocation(tablePath) + CarbonCommonConstants.FILE_SEPARATOR
+ + segmentFile;
+ // Deletes the physical segment file
+ FileFactory.deleteFile(segmentFilePath, FileFactory.getFileType(segmentFilePath));
+ }
+
+ private static void deletePhysicalPartition(List<PartitionSpec> partitionSpecs,
+ Map<String, List<String>> locationMap) {
+ for (Map.Entry<String, List<String>> entry : locationMap.entrySet()) {
+ Path location = new Path(entry.getKey()).getParent();
+ boolean exists = pathExistsInPartitionSpec(partitionSpecs, location);
+ if (!exists) {
+ FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(location.toString()));
+ }
+ }
+ }
+
+ private static boolean pathExistsInPartitionSpec(List<PartitionSpec> partitionSpecs,
+ Path partitionPath) {
+ for (PartitionSpec spec : partitionSpecs) {
+ if (spec.getLocation().equals(partitionPath)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Get the partition specs of the segment
+ * @param segmentId
+ * @param tablePath
+ * @return
+ * @throws IOException
+ */
+ public static List<PartitionSpec> getPartitionSpecs(String segmentId, String tablePath)
+ throws IOException {
+ LoadMetadataDetails segEntry = null;
+ LoadMetadataDetails[] details =
+ SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath));
+ for (LoadMetadataDetails entry : details) {
+ if (entry.getLoadName().equals(segmentId)) {
+ segEntry = entry;
+ break;
+ }
+ }
+ if (segEntry != null && segEntry.getSegmentFile() != null) {
+ SegmentFileStore fileStore = new SegmentFileStore(tablePath, segEntry.getSegmentFile());
+ List<PartitionSpec> partitionSpecs = fileStore.getPartitionSpecs();
+ for (PartitionSpec spec : partitionSpecs) {
+ spec.setUuid(segmentId + "_" + segEntry.getLoadStartTime());
+ }
+ return partitionSpecs;
+ }
+ return null;
+ }
+
+ /**
+ * Move the loaded data from temp folder to respective partition folder.
+ * @param segmentFile
+ * @param tmpFolder
+ * @param tablePath
+ */
+ public static void moveFromTempFolder(SegmentFile segmentFile, String tmpFolder,
+ String tablePath) {
+
+ for (Map.Entry<String, SegmentFileStore.FolderDetails> entry : segmentFile.getLocationMap()
+ .entrySet()) {
+ String location = entry.getKey();
+ if (entry.getValue().isRelative()) {
+ location = tablePath + CarbonCommonConstants.FILE_SEPARATOR + location;
+ }
+ CarbonFile oldFolder =
+ FileFactory.getCarbonFile(location + CarbonCommonConstants.FILE_SEPARATOR + tmpFolder);
+ CarbonFile[] oldFiles = oldFolder.listFiles();
+ for (CarbonFile file : oldFiles) {
+ file.renameForce(location + CarbonCommonConstants.FILE_SEPARATOR + file.getName());
+ }
+ oldFolder.delete();
+ }
+ }
+
+ /**
+ * Remove temp stage folder in case of job aborted.
+ *
+ * @param locationMap
+ * @param tmpFolder
+ * @param tablePath
+ */
+ public static void removeTempFolder(Map<String, FolderDetails> locationMap, String tmpFolder,
+ String tablePath) {
+ if (locationMap == null) {
+ return;
+ }
+ for (Map.Entry<String, SegmentFileStore.FolderDetails> entry : locationMap.entrySet()) {
+ String location = entry.getKey();
+ if (entry.getValue().isRelative()) {
+ location = tablePath + CarbonCommonConstants.FILE_SEPARATOR + location;
+ }
+ CarbonFile oldFolder =
+ FileFactory.getCarbonFile(location + CarbonCommonConstants.FILE_SEPARATOR + tmpFolder);
+ if (oldFolder.exists()) {
+ FileFactory.deleteAllCarbonFilesOfDir(oldFolder);
+ }
+ }
+ }
+
+ /**
+ * Returns content of segment
+ * @return
+ */
+ public Map<String, FolderDetails> getLocationMap() {
+ if (segmentFile == null) {
+ return new HashMap<>();
+ }
+ return segmentFile.getLocationMap();
+ }
+
+ /**
+ * Returs the current partition specs of this segment
+ * @return
+ */
+ public List<PartitionSpec> getPartitionSpecs() {
+ List<PartitionSpec> partitionSpecs = new ArrayList<>();
+ if (segmentFile != null) {
+ for (Map.Entry<String, FolderDetails> entry : segmentFile.getLocationMap().entrySet()) {
+ String location = entry.getKey();
+ if (entry.getValue().isRelative) {
+ location = tablePath + CarbonCommonConstants.FILE_SEPARATOR + location;
+ }
+ if (entry.getValue().getStatus().equals(SegmentStatus.SUCCESS.getMessage())) {
+ partitionSpecs.add(new PartitionSpec(entry.getValue().partitions, location));
+ }
+ }
+ }
+ return partitionSpecs;
+ }
+
+ /**
+ * It contains the segment information like location, partitions and related index files
+ */
+ public static class SegmentFile implements Serializable {
+
+ private static final long serialVersionUID = 3582245668420401089L;
+
+ private Map<String, FolderDetails> locationMap;
+
+ public SegmentFile merge(SegmentFile mapper) {
+ if (this == mapper) {
+ return this;
+ }
+ if (locationMap != null && mapper.locationMap != null) {
+ for (Map.Entry<String, FolderDetails> entry : mapper.locationMap.entrySet()) {
+ FolderDetails folderDetails = locationMap.get(entry.getKey());
+ if (folderDetails != null) {
+ folderDetails.merge(entry.getValue());
+ } else {
+ locationMap.put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ if (locationMap == null) {
+ locationMap = mapper.locationMap;
+ }
+ return this;
+ }
+
+ public Map<String, FolderDetails> getLocationMap() {
+ return locationMap;
+ }
+
+ public void setLocationMap(Map<String, FolderDetails> locationMap) {
+ this.locationMap = locationMap;
+ }
+ }
+
+ /**
+ * Represents one partition folder
+ */
+ public static class FolderDetails implements Serializable {
+
+ private static final long serialVersionUID = 501021868886928553L;
+
+ private Set<String> files = new HashSet<>();
+
+ private List<String> partitions = new ArrayList<>();
+
+ private String status;
+
+ private String mergeFileName;
+
+ private boolean isRelative;
+
+ public FolderDetails merge(FolderDetails folderDetails) {
+ if (this == folderDetails || folderDetails == null) {
+ return this;
+ }
+ if (folderDetails.files != null) {
+ files.addAll(folderDetails.files);
+ }
+ if (files == null) {
+ files = folderDetails.files;
+ }
+ partitions = folderDetails.partitions;
+ return this;
+ }
+
+ public Set<String> getFiles() {
+ return files;
+ }
+
+ public void setFiles(Set<String> files) {
+ this.files = files;
+ }
+
+ public List<String> getPartitions() {
+ return partitions;
+ }
+
+ public void setPartitions(List<String> partitions) {
+ this.partitions = partitions;
+ }
+
+ public boolean isRelative() {
+ return isRelative;
+ }
+
+ public void setRelative(boolean relative) {
+ isRelative = relative;
+ }
+
+ public String getMergeFileName() {
+ return mergeFileName;
+ }
+
+ public void setMergeFileName(String mergeFileName) {
+ this.mergeFileName = mergeFileName;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public void setStatus(String status) {
+ this.status = status;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index c5f61c2..de98fa8 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -82,14 +83,16 @@ public class CarbonUpdateUtil {
* @param factPath
* @return
*/
- public static String getTableBlockPath(String tid, String factPath) {
- String part =
- CarbonTablePath.addPartPrefix(getRequiredFieldFromTID(tid, TupleIdEnum.PART_ID));
+ public static String getTableBlockPath(String tid, String factPath, boolean isPartitionTable) {
+ String partField = getRequiredFieldFromTID(tid, TupleIdEnum.PART_ID);
+ if (isPartitionTable) {
+ return factPath + CarbonCommonConstants.FILE_SEPARATOR + partField;
+ }
+ String part = CarbonTablePath.addPartPrefix(partField);
String segment =
CarbonTablePath.addSegmentPrefix(getRequiredFieldFromTID(tid, TupleIdEnum.SEGMENT_ID));
return factPath + CarbonCommonConstants.FILE_SEPARATOR + part
+ CarbonCommonConstants.FILE_SEPARATOR + segment;
-
}
/**
@@ -172,6 +175,22 @@ public class CarbonUpdateUtil {
}
/**
+ * Update table status
+ * @param updatedSegmentsList
+ * @param table
+ * @param updatedTimeStamp
+ * @param isTimestampUpdationRequired
+ * @param segmentsToBeDeleted
+ * @return
+ */
+ public static boolean updateTableMetadataStatus(Set<Segment> updatedSegmentsList,
+ CarbonTable table, String updatedTimeStamp, boolean isTimestampUpdationRequired,
+ List<Segment> segmentsToBeDeleted) {
+ return updateTableMetadataStatus(updatedSegmentsList, table, updatedTimeStamp,
+ isTimestampUpdationRequired, segmentsToBeDeleted, new ArrayList<Segment>());
+ }
+
+ /**
*
* @param updatedSegmentsList
* @param table
@@ -180,10 +199,9 @@ public class CarbonUpdateUtil {
* @param segmentsToBeDeleted
* @return
*/
- public static boolean updateTableMetadataStatus(Set<String> updatedSegmentsList,
- CarbonTable table, String updatedTimeStamp,
- boolean isTimestampUpdationRequired,
- List<String> segmentsToBeDeleted) {
+ public static boolean updateTableMetadataStatus(Set<Segment> updatedSegmentsList,
+ CarbonTable table, String updatedTimeStamp, boolean isTimestampUpdationRequired,
+ List<Segment> segmentsToBeDeleted, List<Segment> segmentFilesTobeUpdated) {
boolean status = false;
@@ -221,13 +239,13 @@ public class CarbonUpdateUtil {
}
// if the segments is in the list of marked for delete then update the status.
- if (segmentsToBeDeleted.contains(loadMetadata.getLoadName())) {
+ if (segmentsToBeDeleted.contains(new Segment(loadMetadata.getLoadName(), null))) {
loadMetadata.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
loadMetadata.setModificationOrdeletionTimesStamp(Long.parseLong(updatedTimeStamp));
}
}
- for (String segName : updatedSegmentsList) {
- if (loadMetadata.getLoadName().equalsIgnoreCase(segName)) {
+ for (Segment segName : updatedSegmentsList) {
+ if (loadMetadata.getLoadName().equalsIgnoreCase(segName.getSegmentNo())) {
// if this call is coming from the delete delta flow then the time stamp
// String will come empty then no need to write into table status file.
if (isTimestampUpdationRequired) {
@@ -240,6 +258,10 @@ public class CarbonUpdateUtil {
// update end timestamp for each time.
loadMetadata.setUpdateDeltaEndTimestamp(updatedTimeStamp);
}
+ if (segmentFilesTobeUpdated.contains(Segment.toSegment(loadMetadata.getLoadName()))) {
+ loadMetadata.setSegmentFile(loadMetadata.getLoadName() + "_" + updatedTimeStamp
+ + CarbonTablePath.SEGMENT_EXT);
+ }
}
}
}
@@ -696,14 +718,14 @@ public class CarbonUpdateUtil {
* @param segmentBlockCount
* @return
*/
- public static List<String> getListOfSegmentsToMarkDeleted(Map<String, Long> segmentBlockCount) {
- List<String> segmentsToBeDeleted =
+ public static List<Segment> getListOfSegmentsToMarkDeleted(Map<String, Long> segmentBlockCount) {
+ List<Segment> segmentsToBeDeleted =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
for (Map.Entry<String, Long> eachSeg : segmentBlockCount.entrySet()) {
if (eachSeg.getValue() == 0) {
- segmentsToBeDeleted.add(eachSeg.getKey());
+ segmentsToBeDeleted.add(new Segment(eachSeg.getKey(), null));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 6490694..cc2e513 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -74,7 +74,6 @@ import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
-import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.commons.lang3.ArrayUtils;
@@ -269,7 +268,8 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
blockExecutionInfoList.add(getBlockExecutionInfoForBlock(queryModel, abstractIndex,
dataRefNode.getBlockInfos().get(0).getBlockletInfos().getStartBlockletNumber(),
dataRefNode.numberOfNodes(), dataRefNode.getBlockInfos().get(0).getFilePath(),
- dataRefNode.getBlockInfos().get(0).getDeletedDeltaFilePath()));
+ dataRefNode.getBlockInfos().get(0).getDeletedDeltaFilePath(),
+ dataRefNode.getBlockInfos().get(0).getSegmentId()));
}
if (null != queryModel.getStatisticsRecorder()) {
QueryStatistic queryStatistic = new QueryStatistic();
@@ -291,7 +291,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
*/
protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath,
- String[] deleteDeltaFiles)
+ String[] deleteDeltaFiles, String segmentId)
throws QueryExecutionException {
BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
@@ -304,11 +304,8 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo,
queryModel.getQueryDimension(), tableBlockDimensions,
segmentProperties.getComplexDimensions());
- int tableFactPathLength = CarbonStorePath
- .getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getTablePath(),
- queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier()).getFactDir()
- .length() + 1;
- blockExecutionInfo.setBlockId(filePath.substring(tableFactPathLength));
+ blockExecutionInfo.setBlockId(
+ CarbonUtil.getBlockId(queryModel.getAbsoluteTableIdentifier(), filePath, segmentId));
blockExecutionInfo.setDeleteDeltaFilePath(deleteDeltaFiles);
blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
@@ -457,6 +454,8 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
return blockExecutionInfo;
}
+
+
/**
* This method will be used to get fixed key length size this will be used
* to create a row from column chunk
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index 85602bc..a0fa67d 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -123,6 +123,11 @@ public class LoadMetadataDetails implements Serializable {
*/
private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
+ /**
+ * Segment file name where it has the information of partition information.
+ */
+ private String segmentFile;
+
public String getPartitionCount() {
return partitionCount;
}
@@ -417,4 +422,17 @@ public class LoadMetadataDetails implements Serializable {
public void setFileFormat(FileFormat fileFormat) {
this.fileFormat = fileFormat;
}
+
+ public String getSegmentFile() {
+ return segmentFile;
+ }
+
+ public void setSegmentFile(String segmentFile) {
+ this.segmentFile = segmentFile;
+ }
+
+ @Override public String toString() {
+ return "LoadMetadataDetails{" + "loadStatus=" + loadStatus + ", loadName='" + loadName + '\''
+ + ", loadStartTime='" + loadStartTime + '\'' + ", segmentFile='" + segmentFile + '\'' + '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index 9d14c62..76c2dc7 100755
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -33,6 +33,7 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
@@ -99,13 +100,14 @@ public class SegmentStatusManager {
public ValidAndInvalidSegmentsInfo getValidAndInvalidSegments() throws IOException {
// @TODO: move reading LoadStatus file to separate class
- List<String> listOfValidSegments = new ArrayList<>(10);
- List<String> listOfValidUpdatedSegments = new ArrayList<>(10);
- List<String> listOfInvalidSegments = new ArrayList<>(10);
- List<String> listOfStreamSegments = new ArrayList<>(10);
+ List<Segment> listOfValidSegments = new ArrayList<>(10);
+ List<Segment> listOfValidUpdatedSegments = new ArrayList<>(10);
+ List<Segment> listOfInvalidSegments = new ArrayList<>(10);
+ List<Segment> listOfStreamSegments = new ArrayList<>(10);
+ List<Segment> listOfInProgressSegments = new ArrayList<>(10);
CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
+ .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
+ absoluteTableIdentifier.getCarbonTableIdentifier());
String dataPath = carbonTablePath.getTableStatusFilePath();
DataInputStream dataInputStream = null;
@@ -113,7 +115,7 @@ public class SegmentStatusManager {
Gson gson = new Gson();
AtomicFileOperations fileOperation =
- new AtomicFileOperationsImpl(dataPath, FileFactory.getFileType(dataPath));
+ new AtomicFileOperationsImpl(dataPath, FileFactory.getFileType(dataPath));
LoadMetadataDetails[] loadFolderDetailsArray;
try {
if (FileFactory.isFileExist(dataPath, FileFactory.getFileType(dataPath))) {
@@ -127,37 +129,44 @@ public class SegmentStatusManager {
}
//just directly iterate Array
for (LoadMetadataDetails segment : loadFolderDetailsArray) {
- if (SegmentStatus.SUCCESS == segment.getSegmentStatus() ||
- SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus() ||
- SegmentStatus.LOAD_PARTIAL_SUCCESS == segment.getSegmentStatus() ||
- SegmentStatus.STREAMING == segment.getSegmentStatus() ||
- SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) {
+ if (SegmentStatus.SUCCESS == segment.getSegmentStatus()
+ || SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus()
+ || SegmentStatus.LOAD_PARTIAL_SUCCESS == segment.getSegmentStatus()
+ || SegmentStatus.STREAMING == segment.getSegmentStatus()
+ || SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) {
// check for merged loads.
if (null != segment.getMergedLoadName()) {
- if (!listOfValidSegments.contains(segment.getMergedLoadName())) {
- listOfValidSegments.add(segment.getMergedLoadName());
+ Segment seg = new Segment(segment.getMergedLoadName(), segment.getSegmentFile());
+ if (!listOfValidSegments.contains(seg)) {
+ listOfValidSegments.add(seg);
}
// if merged load is updated then put it in updated list
if (SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus()) {
- listOfValidUpdatedSegments.add(segment.getMergedLoadName());
+ listOfValidUpdatedSegments.add(seg);
}
continue;
}
if (SegmentStatus.MARKED_FOR_UPDATE == segment.getSegmentStatus()) {
- listOfValidUpdatedSegments.add(segment.getLoadName());
+ listOfValidUpdatedSegments
+ .add(new Segment(segment.getLoadName(), segment.getSegmentFile()));
}
- if (SegmentStatus.STREAMING == segment.getSegmentStatus() ||
- SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) {
- listOfStreamSegments.add(segment.getLoadName());
+ if (SegmentStatus.STREAMING == segment.getSegmentStatus()
+ || SegmentStatus.STREAMING_FINISH == segment.getSegmentStatus()) {
+ listOfStreamSegments
+ .add(new Segment(segment.getLoadName(), segment.getSegmentFile()));
continue;
}
- listOfValidSegments.add(segment.getLoadName());
- } else if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus() ||
- SegmentStatus.COMPACTED == segment.getSegmentStatus() ||
- SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) {
- listOfInvalidSegments.add(segment.getLoadName());
+ listOfValidSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile()));
+ } else if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus()
+ || SegmentStatus.COMPACTED == segment.getSegmentStatus()
+ || SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) {
+ listOfInvalidSegments.add(new Segment(segment.getLoadName(), segment.getSegmentFile()));
+ } else if (SegmentStatus.INSERT_IN_PROGRESS == segment.getSegmentStatus() ||
+ SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segment.getSegmentStatus()) {
+ listOfInProgressSegments
+ .add(new Segment(segment.getLoadName(), segment.getSegmentFile()));
}
}
}
@@ -168,7 +177,7 @@ public class SegmentStatusManager {
CarbonUtil.closeStreams(dataInputStream);
}
return new ValidAndInvalidSegmentsInfo(listOfValidSegments, listOfValidUpdatedSegments,
- listOfInvalidSegments, listOfStreamSegments);
+ listOfInvalidSegments, listOfStreamSegments, listOfInProgressSegments);
}
/**
@@ -688,29 +697,35 @@ public class SegmentStatusManager {
public static class ValidAndInvalidSegmentsInfo {
- private final List<String> listOfValidSegments;
- private final List<String> listOfValidUpdatedSegments;
- private final List<String> listOfInvalidSegments;
- private final List<String> listOfStreamSegments;
-
- private ValidAndInvalidSegmentsInfo(List<String> listOfValidSegments,
- List<String> listOfValidUpdatedSegments, List<String> listOfInvalidUpdatedSegments,
- List<String> listOfStreamSegments) {
+ private final List<Segment> listOfValidSegments;
+ private final List<Segment> listOfValidUpdatedSegments;
+ private final List<Segment> listOfInvalidSegments;
+ private final List<Segment> listOfStreamSegments;
+ private final List<Segment> listOfInProgressSegments;
+
+ private ValidAndInvalidSegmentsInfo(List<Segment> listOfValidSegments,
+ List<Segment> listOfValidUpdatedSegments, List<Segment> listOfInvalidUpdatedSegments,
+ List<Segment> listOfStreamSegments, List<Segment> listOfInProgressSegments) {
this.listOfValidSegments = listOfValidSegments;
this.listOfValidUpdatedSegments = listOfValidUpdatedSegments;
this.listOfInvalidSegments = listOfInvalidUpdatedSegments;
this.listOfStreamSegments = listOfStreamSegments;
+ this.listOfInProgressSegments = listOfInProgressSegments;
}
- public List<String> getInvalidSegments() {
+ public List<Segment> getInvalidSegments() {
return listOfInvalidSegments;
}
- public List<String> getValidSegments() {
+ public List<Segment> getValidSegments() {
return listOfValidSegments;
}
- public List<String> getStreamSegments() {
+ public List<Segment> getStreamSegments() {
return listOfStreamSegments;
}
+
+ public List<Segment> getListOfInProgressSegments() {
+ return listOfInProgressSegments;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index e0e7b70..71b6ba8 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -33,6 +33,7 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -70,6 +71,7 @@ public class SegmentUpdateStatusManager {
private SegmentUpdateDetails[] updateDetails;
private CarbonTablePath carbonTablePath;
private Map<String, SegmentUpdateDetails> blockAndDetailsMap;
+ private boolean isPartitionTable;
/**
* @param absoluteTableIdentifier
@@ -83,6 +85,9 @@ public class SegmentUpdateStatusManager {
// on latest file status.
segmentDetails =
segmentStatusManager.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
+ if (segmentDetails.length > 0) {
+ isPartitionTable = segmentDetails[0].getSegmentFile() != null;
+ }
updateDetails = readLoadMetadata();
populateMap();
}
@@ -268,12 +273,14 @@ public class SegmentUpdateStatusManager {
* @return all delete delta files
* @throws Exception
*/
- public String[] getDeleteDeltaFilePath(String blockFilePath) throws Exception {
- int tableFactPathLength = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier()).getFactDir().length() + 1;
- String blockame = blockFilePath.substring(tableFactPathLength);
- String tupleId = CarbonTablePath.getShortBlockId(blockame);
+ public String[] getDeleteDeltaFilePath(String blockFilePath, String segmentId) throws Exception {
+ String blockId = CarbonUtil.getBlockId(absoluteTableIdentifier, blockFilePath, segmentId);
+ String tupleId;
+ if (isPartitionTable) {
+ tupleId = CarbonTablePath.getShortBlockIdForPartitionTable(blockId);
+ } else {
+ tupleId = CarbonTablePath.getShortBlockId(blockId);
+ }
return getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT)
.toArray(new String[0]);
}
@@ -292,12 +299,19 @@ public class SegmentUpdateStatusManager {
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
String segment = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID);
- String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment);
String completeBlockName = CarbonTablePath.addDataPartPrefix(
CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCK_ID)
+ CarbonCommonConstants.FACT_FILE_EXT);
- String blockPath =
- carbonDataDirectoryPath + CarbonCommonConstants.FILE_SEPARATOR + completeBlockName;
+ String blockPath;
+ if (isPartitionTable) {
+ blockPath = absoluteTableIdentifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR
+ + CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.PART_ID)
+ .replace("#", "/") + CarbonCommonConstants.FILE_SEPARATOR + completeBlockName;
+ } else {
+ String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment);
+ blockPath =
+ carbonDataDirectoryPath + CarbonCommonConstants.FILE_SEPARATOR + completeBlockName;
+ }
CarbonFile file = FileFactory.getCarbonFile(blockPath, FileFactory.getFileType(blockPath));
if (!file.exists()) {
throw new Exception("Invalid tuple id " + tupleId);
@@ -306,8 +320,7 @@ public class SegmentUpdateStatusManager {
//blockName without timestamp
final String blockNameFromTuple =
blockNameWithoutExtn.substring(0, blockNameWithoutExtn.lastIndexOf("-"));
- return getDeltaFiles(file, blockNameFromTuple, extension,
- segment);
+ return getDeltaFiles(file, blockNameFromTuple, extension, segment);
} catch (Exception ex) {
String errorMsg = "Invalid tuple id " + tupleId;
LOG.error(errorMsg);
@@ -418,20 +431,20 @@ public class SegmentUpdateStatusManager {
* @param blockName
* @return
*/
- public CarbonFile[] getDeleteDeltaFilesList(final String segmentId, final String blockName) {
+ public CarbonFile[] getDeleteDeltaFilesList(final Segment segmentId, final String blockName) {
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId.getSegmentNo());
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
for (SegmentUpdateDetails block : updateDetails) {
if ((block.getBlockName().equalsIgnoreCase(blockName)) &&
- (block.getSegmentName().equalsIgnoreCase(segmentId))
+ (block.getSegmentName().equalsIgnoreCase(segmentId.getSegmentNo()))
&& !CarbonUpdateUtil.isBlockInvalid((block.getSegmentStatus()))) {
final long deltaStartTimestamp =
getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index c208154..eb0a9d7 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -47,6 +47,7 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -63,6 +64,7 @@ import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
@@ -1437,14 +1439,13 @@ public final class CarbonUtil {
* @param values
* @return comma separated segment string
*/
- public static String convertToString(List<String> values) {
+ public static String convertToString(List<Segment> values) {
if (values == null || values.isEmpty()) {
return "";
}
StringBuilder segmentStringbuilder = new StringBuilder();
for (int i = 0; i < values.size() - 1; i++) {
- String segmentNo = values.get(i);
- segmentStringbuilder.append(segmentNo);
+ segmentStringbuilder.append(values.get(i));
segmentStringbuilder.append(",");
}
segmentStringbuilder.append(values.get(values.size() - 1));
@@ -2145,7 +2146,8 @@ public final class CarbonUtil {
}
for (String value : values) {
if (!value.equalsIgnoreCase("*")) {
- Float aFloatValue = Float.parseFloat(value);
+ Segment segment = Segment.toSegment(value);
+ Float aFloatValue = Float.parseFloat(segment.getSegmentNo());
if (aFloatValue < 0 || aFloatValue > Float.MAX_VALUE) {
throw new InvalidConfigurationException(
"carbon.input.segments.<database_name>.<table_name> value range should be greater "
@@ -2298,7 +2300,7 @@ public final class CarbonUtil {
}
// Get the total size of carbon data and the total size of carbon index
- public static HashMap<String, Long> getDataSizeAndIndexSize(CarbonTablePath carbonTablePath,
+ private static HashMap<String, Long> getDataSizeAndIndexSize(CarbonTablePath carbonTablePath,
String segmentId) throws IOException {
long carbonDataSize = 0L;
long carbonIndexSize = 0L;
@@ -2350,6 +2352,51 @@ public final class CarbonUtil {
return dataAndIndexSize;
}
+ // Get the total size of carbon data and the total size of carbon index
+ private static HashMap<String, Long> getDataSizeAndIndexSize(SegmentFileStore fileStore)
+ throws IOException {
+ long carbonDataSize = 0L;
+ long carbonIndexSize = 0L;
+ HashMap<String, Long> dataAndIndexSize = new HashMap<String, Long>();
+ if (fileStore.getLocationMap() != null) {
+ fileStore.readIndexFiles();
+ Map<String, List<String>> indexFilesMap = fileStore.getIndexFilesMap();
+ for (Map.Entry<String, List<String>> entry : indexFilesMap.entrySet()) {
+ carbonIndexSize += FileFactory.getCarbonFile(entry.getKey()).getSize();
+ for (String blockFile : entry.getValue()) {
+ carbonDataSize += FileFactory.getCarbonFile(blockFile).getSize();
+ }
+ }
+ }
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE, carbonDataSize);
+ dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, carbonIndexSize);
+ return dataAndIndexSize;
+ }
+
+ // Get the total size of carbon data and the total size of carbon index
+ public static HashMap<String, Long> getDataSizeAndIndexSize(CarbonTablePath carbonTablePath,
+ Segment segment) throws IOException {
+ if (segment.getSegmentFileName() != null) {
+ SegmentFileStore fileStore =
+ new SegmentFileStore(carbonTablePath.getPath(), segment.getSegmentFileName());
+ return getDataSizeAndIndexSize(fileStore);
+ } else {
+ return getDataSizeAndIndexSize(carbonTablePath, segment.getSegmentNo());
+ }
+ }
+
+ // Get the total size of segment.
+ public static long getSizeOfSegment(CarbonTablePath carbonTablePath,
+ Segment segment) throws IOException {
+ HashMap<String, Long> dataSizeAndIndexSize = getDataSizeAndIndexSize(carbonTablePath, segment);
+ long size = 0;
+ for (Long eachSize: dataSizeAndIndexSize.values()) {
+ size += eachSize;
+ }
+ return size;
+ }
+
+
/**
* Utility function to check whether table has timseries datamap or not
* @param carbonTable
@@ -2449,5 +2496,43 @@ public final class CarbonUtil {
return updatedMinMaxValues;
}
+ /**
+ * Generate the blockid as per the block path
+ *
+ * @param identifier
+ * @param filePath
+ * @param segmentId
+ * @return
+ */
+ public static String getBlockId(AbsoluteTableIdentifier identifier, String filePath,
+ String segmentId) {
+ String blockId;
+ String blockName = filePath.substring(filePath.lastIndexOf("/") + 1, filePath.length());
+ String tablePath = identifier.getTablePath();
+ if (filePath.startsWith(tablePath)) {
+ String factDir =
+ CarbonStorePath.getCarbonTablePath(tablePath, identifier.getCarbonTableIdentifier())
+ .getFactDir();
+ if (filePath.startsWith(factDir)) {
+ blockId = "Part0" + CarbonCommonConstants.FILE_SEPARATOR + "Segment_" + segmentId
+ + CarbonCommonConstants.FILE_SEPARATOR + blockName;
+ } else {
+ // This is the case with partition table.
+ String partitionDir =
+ filePath.substring(tablePath.length() + 1, filePath.length() - blockName.length() - 1);
+
+ // Replace / with # on partition director to support multi level partitioning. And access
+ // them all as a single entity.
+ blockId = partitionDir.replace("/", "#") + CarbonCommonConstants.FILE_SEPARATOR + "Segment_"
+ + segmentId + CarbonCommonConstants.FILE_SEPARATOR + blockName;
+ }
+ } else {
+ blockId = filePath.substring(0, filePath.length() - blockName.length()).replace("/", "#")
+ + CarbonCommonConstants.FILE_SEPARATOR + "Segment_" + segmentId
+ + CarbonCommonConstants.FILE_SEPARATOR + blockName;
+ }
+ return blockId;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java
index d4b328d..f1603dc 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeConverterImpl.java
@@ -18,6 +18,7 @@
package org.apache.carbondata.core.util;
import java.io.Serializable;
+import java.math.BigDecimal;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -26,18 +27,33 @@ public class DataTypeConverterImpl implements DataTypeConverter, Serializable {
private static final long serialVersionUID = -1718154403432354200L;
public Object convertToDecimal(Object data) {
- return new java.math.BigDecimal(data.toString());
+ if (null == data) {
+ return null;
+ }
+ if (data instanceof BigDecimal) {
+ return data;
+ }
+ return new BigDecimal(data.toString());
}
public Object convertFromByteToUTF8String(Object data) {
+ if (null == data) {
+ return null;
+ }
return new String((byte[]) data, CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
}
public byte[] convertFromStringToByte(Object data) {
+ if (null == data) {
+ return null;
+ }
return data.toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
}
public Object convertFromStringToUTF8String(Object data) {
+ if (null == data) {
+ return null;
+ }
return data.toString();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index c370b14..4602cc4 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -83,25 +83,7 @@ public final class DataTypeUtil {
*/
public static Object getMeasureValueBasedOnDataType(String msrValue, DataType dataType,
CarbonMeasure carbonMeasure) {
- if (dataType == DataTypes.BOOLEAN) {
- return BooleanConvert.parseBoolean(msrValue);
- } else if (DataTypes.isDecimal(dataType)) {
- BigDecimal bigDecimal =
- new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
- return normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision());
- } else if (dataType == DataTypes.SHORT) {
- return Short.parseShort(msrValue);
- } else if (dataType == DataTypes.INT) {
- return Integer.parseInt(msrValue);
- } else if (dataType == DataTypes.LONG) {
- return Long.valueOf(msrValue);
- } else {
- Double parsedValue = Double.valueOf(msrValue);
- if (Double.isInfinite(parsedValue) || Double.isNaN(parsedValue)) {
- return null;
- }
- return parsedValue;
- }
+ return getMeasureValueBasedOnDataType(msrValue, dataType,carbonMeasure, false);
}
/**
@@ -112,15 +94,19 @@ public final class DataTypeUtil {
* @param carbonMeasure
* @return
*/
- public static Object getConvertedMeasureValueBasedOnDataType(String msrValue, DataType dataType,
- CarbonMeasure carbonMeasure) {
+ public static Object getMeasureValueBasedOnDataType(String msrValue, DataType dataType,
+ CarbonMeasure carbonMeasure, boolean useConverter) {
if (dataType == DataTypes.BOOLEAN) {
return BooleanConvert.parseBoolean(msrValue);
} else if (DataTypes.isDecimal(dataType)) {
BigDecimal bigDecimal =
new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
- return converter
- .convertToDecimal(normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision()));
+ BigDecimal decimal = normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision());
+ if (useConverter) {
+ return converter.convertToDecimal(decimal);
+ } else {
+ return decimal;
+ }
} else if (dataType == DataTypes.SHORT) {
return Short.parseShort(msrValue);
} else if (dataType == DataTypes.INT) {
@@ -815,6 +801,8 @@ public final class DataTypeUtil {
*/
public static void setDataTypeConverter(DataTypeConverter converterLocal) {
converter = converterLocal;
+ timeStampformatter.remove();
+ dateformatter.remove();
}
public static DataTypeConverter getDataTypeConverter() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1997ca23/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 5a63d2f..d70d9ef 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -48,7 +48,7 @@ public class CarbonTablePath extends Path {
public static final String CARBON_DATA_EXT = ".carbondata";
public static final String INDEX_FILE_EXT = ".carbonindex";
public static final String MERGE_INDEX_FILE_EXT = ".carbonindexmerge";
- public static final String PARTITION_MAP_EXT = ".partitionmap";
+ public static final String SEGMENT_EXT = ".segment";
private static final String STREAMING_DIR = ".streaming";
private static final String STREAMING_LOG_DIR = "log";
@@ -111,17 +111,6 @@ public class CarbonTablePath extends Path {
}
/**
- * Return true if the fileNameWithPath ends with partition map file extension name
- */
- public static boolean isPartitionMapFile(String fileNameWithPath) {
- int pos = fileNameWithPath.lastIndexOf('.');
- if (pos != -1) {
- return fileNameWithPath.substring(pos).startsWith(PARTITION_MAP_EXT);
- }
- return false;
- }
-
- /**
* check if it is carbon index file matching extension
*
* @param fileNameWithPath
@@ -667,6 +656,18 @@ public class CarbonTablePath extends Path {
}
/**
+ * This method will remove strings in path and return short block id
+ *
+ * @param blockId
+ * @return shortBlockId
+ */
+ public static String getShortBlockIdForPartitionTable(String blockId) {
+ return blockId.replace(SEGMENT_PREFIX, "")
+ .replace(DATA_PART_PREFIX, "")
+ .replace(CARBON_DATA_EXT, "");
+ }
+
+ /**
* This method will append strings in path and return block id
*
* @param shortBlockId
@@ -735,4 +736,12 @@ public class CarbonTablePath extends Path {
public static String getSegmentPath(String tablePath, String segmentId) {
return tablePath + "/Fact/Part0/Segment_" + segmentId;
}
+
+ /**
+ * Get the segment file locations of table
+ */
+ public static String getSegmentFilesLocation(String tablePath) {
+ return getMetadataPath(tablePath) + CarbonCommonConstants.FILE_SEPARATOR + "segments";
+ }
+
}
\ No newline at end of file