You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/12/17 14:16:43 UTC
[01/21] carbondata git commit: [CARBONDATA-3145] Avoid duplicate
decoding for complex column pages while querying
Repository: carbondata
Updated Branches:
refs/heads/branch-1.5 442e2446e -> 951a7811a
[CARBONDATA-3145] Avoid duplicate decoding for complex column pages while querying
Problem:
Column page is decoded for getting each row of a complex primitive column.
Solution:
Decode a page it once then use the same.
This closes #2975
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/523515cd
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/523515cd
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/523515cd
Branch: refs/heads/branch-1.5
Commit: 523515cd6700f0c19d43de126525fd3c8dede0b9
Parents: d909482
Author: dhatchayani <dh...@gmail.com>
Authored: Wed Dec 5 12:40:56 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:50:23 2018 +0530
----------------------------------------------------------------------
.../core/scan/complextypes/ArrayQueryType.java | 11 ++--
.../scan/complextypes/ComplexQueryType.java | 14 +++-
.../scan/complextypes/PrimitiveQueryType.java | 11 ++--
.../core/scan/complextypes/StructQueryType.java | 14 ++--
.../core/scan/filter/GenericQueryType.java | 4 +-
.../executer/RowLevelFilterExecuterImpl.java | 7 +-
.../core/scan/result/BlockletScannedResult.java | 68 +++++++++++++-------
7 files changed, 86 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/523515cd/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
index a5f4234..8538edb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Map;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
@@ -62,17 +63,17 @@ public class ArrayQueryType extends ComplexQueryType implements GenericQueryType
}
public void parseBlocksAndReturnComplexColumnByteArray(DimensionRawColumnChunk[] rawColumnChunks,
- int rowNumber, int pageNumber, DataOutputStream dataOutputStream) throws IOException {
- byte[] input = copyBlockDataChunk(rawColumnChunks, rowNumber, pageNumber);
+ DimensionColumnPage[][] dimensionColumnPages, int rowNumber, int pageNumber,
+ DataOutputStream dataOutputStream) throws IOException {
+ byte[] input = copyBlockDataChunk(rawColumnChunks, dimensionColumnPages, rowNumber, pageNumber);
ByteBuffer byteArray = ByteBuffer.wrap(input);
int dataLength = byteArray.getInt();
dataOutputStream.writeInt(dataLength);
if (dataLength > 0) {
int dataOffset = byteArray.getInt();
for (int i = 0; i < dataLength; i++) {
- children
- .parseBlocksAndReturnComplexColumnByteArray(rawColumnChunks, dataOffset++, pageNumber,
- dataOutputStream);
+ children.parseBlocksAndReturnComplexColumnByteArray(rawColumnChunks, dimensionColumnPages,
+ dataOffset++, pageNumber, dataOutputStream);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/523515cd/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
index 98f0715..704af89 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.scan.complextypes;
import java.io.IOException;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
@@ -40,9 +41,10 @@ public class ComplexQueryType {
* This method is also used by child.
*/
protected byte[] copyBlockDataChunk(DimensionRawColumnChunk[] rawColumnChunks,
- int rowNumber, int pageNumber) {
+ DimensionColumnPage[][] dimensionColumnPages, int rowNumber, int pageNumber) {
byte[] data =
- rawColumnChunks[blockIndex].decodeColumnPage(pageNumber).getChunkData(rowNumber);
+ getDecodedDimensionPage(dimensionColumnPages, rawColumnChunks[blockIndex], pageNumber)
+ .getChunkData(rowNumber);
byte[] output = new byte[data.length];
System.arraycopy(data, 0, output, 0, output.length);
return output;
@@ -57,4 +59,12 @@ public class ComplexQueryType {
.readDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
}
+
+ private DimensionColumnPage getDecodedDimensionPage(DimensionColumnPage[][] dimensionColumnPages,
+ DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) {
+ if (dimensionColumnPages == null || null == dimensionColumnPages[blockIndex]) {
+ return dimensionRawColumnChunk.decodeColumnPage(pageNumber);
+ }
+ return dimensionColumnPages[blockIndex][pageNumber];
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/523515cd/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
index abe33c4..6347397 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
@@ -93,10 +94,12 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
return 1;
}
- @Override public void parseBlocksAndReturnComplexColumnByteArray(
- DimensionRawColumnChunk[] rawColumnChunks, int rowNumber,
- int pageNumber, DataOutputStream dataOutputStream) throws IOException {
- byte[] currentVal = copyBlockDataChunk(rawColumnChunks, rowNumber, pageNumber);
+ @Override
+ public void parseBlocksAndReturnComplexColumnByteArray(DimensionRawColumnChunk[] rawColumnChunks,
+ DimensionColumnPage[][] dimensionColumnPages, int rowNumber, int pageNumber,
+ DataOutputStream dataOutputStream) throws IOException {
+ byte[] currentVal =
+ copyBlockDataChunk(rawColumnChunks, dimensionColumnPages, rowNumber, pageNumber);
if (!this.isDictionary && !this.isDirectDictionary) {
dataOutputStream.writeShort(currentVal.length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/523515cd/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
index c607f84..7bccbc0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
@@ -79,17 +80,18 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
}
@Override public void parseBlocksAndReturnComplexColumnByteArray(
- DimensionRawColumnChunk[] dimensionColumnDataChunks, int rowNumber,
- int pageNumber, DataOutputStream dataOutputStream) throws IOException {
- byte[] input = copyBlockDataChunk(dimensionColumnDataChunks, rowNumber, pageNumber);
+ DimensionRawColumnChunk[] dimensionColumnDataChunks,
+ DimensionColumnPage[][] dimensionColumnPages, int rowNumber, int pageNumber,
+ DataOutputStream dataOutputStream) throws IOException {
+ byte[] input =
+ copyBlockDataChunk(dimensionColumnDataChunks, dimensionColumnPages, rowNumber, pageNumber);
ByteBuffer byteArray = ByteBuffer.wrap(input);
int childElement = byteArray.getShort();
dataOutputStream.writeShort(childElement);
if (childElement > 0) {
for (int i = 0; i < childElement; i++) {
- children.get(i)
- .parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks, rowNumber,
- pageNumber, dataOutputStream);
+ children.get(i).parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks,
+ dimensionColumnPages, rowNumber, pageNumber, dataOutputStream);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/523515cd/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
index 6c087d7..b43062e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Map;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
@@ -41,7 +42,8 @@ public interface GenericQueryType {
int getColsCount();
void parseBlocksAndReturnComplexColumnByteArray(DimensionRawColumnChunk[] rawColumnChunks,
- int rowNumber, int pageNumber, DataOutputStream dataOutputStream) throws IOException;
+ DimensionColumnPage[][] dimensionColumnPages, int rowNumber, int pageNumber,
+ DataOutputStream dataOutputStream) throws IOException;
void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder) throws IOException;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/523515cd/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 7ca2579..63ae0cd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -457,9 +457,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
complexType.parseBlocksAndReturnComplexColumnByteArray(
- blockChunkHolder.getDimensionRawColumnChunks(), index, pageIndex, dataOutputStream);
- record[dimColumnEvaluatorInfo.getRowIndex()] = complexType
- .getDataBasedOnDataType(ByteBuffer.wrap(byteStream.toByteArray()));
+ blockChunkHolder.getDimensionRawColumnChunks(), null, index, pageIndex,
+ dataOutputStream);
+ record[dimColumnEvaluatorInfo.getRowIndex()] =
+ complexType.getDataBasedOnDataType(ByteBuffer.wrap(byteStream.toByteArray()));
byteStream.close();
} catch (IOException e) {
LOGGER.info(e.getMessage());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/523515cd/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
index bb373eb..c04df52 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.core.scan.result;
import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
@@ -46,6 +45,7 @@ import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
import org.apache.carbondata.core.stats.QueryStatisticsModel;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.ReUsableByteArrayDataOutputStream;
import org.apache.log4j.Logger;
@@ -282,30 +282,38 @@ public abstract class BlockletScannedResult {
}
public void fillColumnarComplexBatch(ColumnVectorInfo[] vectorInfos) {
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ ReUsableByteArrayDataOutputStream reuseableDataOutput =
+ new ReUsableByteArrayDataOutputStream(byteStream);
+ boolean isExceptionThrown = false;
for (int i = 0; i < vectorInfos.length; i++) {
int offset = vectorInfos[i].offset;
int len = offset + vectorInfos[i].size;
int vectorOffset = vectorInfos[i].vectorOffset;
CarbonColumnVector vector = vectorInfos[i].vector;
for (int j = offset; j < len; j++) {
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutput = new DataOutputStream(byteStream);
try {
- vectorInfos[i].genericQueryType.parseBlocksAndReturnComplexColumnByteArray(
- dimRawColumnChunks,
- pageFilteredRowId == null ? j : pageFilteredRowId[pageCounter][j], pageCounter,
- dataOutput);
+ vectorInfos[i].genericQueryType
+ .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, dimensionColumnPages,
+ pageFilteredRowId == null ? j : pageFilteredRowId[pageCounter][j], pageCounter,
+ reuseableDataOutput);
Object data = vectorInfos[i].genericQueryType
- .getDataBasedOnDataType(ByteBuffer.wrap(byteStream.toByteArray()));
+ .getDataBasedOnDataType(ByteBuffer.wrap(reuseableDataOutput.getByteArray()));
vector.putObject(vectorOffset++, data);
+ reuseableDataOutput.reset();
} catch (IOException e) {
+ isExceptionThrown = true;
LOGGER.error(e);
} finally {
- CarbonUtil.closeStreams(dataOutput);
- CarbonUtil.closeStreams(byteStream);
+ if (isExceptionThrown) {
+ CarbonUtil.closeStreams(reuseableDataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
}
}
}
+ CarbonUtil.closeStreams(reuseableDataOutput);
+ CarbonUtil.closeStreams(byteStream);
}
/**
@@ -541,6 +549,10 @@ public abstract class BlockletScannedResult {
*/
protected List<byte[][]> getComplexTypeKeyArrayBatch() {
List<byte[][]> complexTypeArrayList = new ArrayList<>(validRowIds.size());
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ ReUsableByteArrayDataOutputStream reUseableDataOutput =
+ new ReUsableByteArrayDataOutputStream(byteStream);
+ boolean isExceptionThrown = false;
byte[][] complexTypeData = null;
// everyTime it is initialized new as in case of prefetch it can modify the data
for (int i = 0; i < validRowIds.size(); i++) {
@@ -552,23 +564,27 @@ public abstract class BlockletScannedResult {
GenericQueryType genericQueryType =
complexParentIndexToQueryMap.get(complexParentBlockIndexes[i]);
for (int j = 0; j < validRowIds.size(); j++) {
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutput = new DataOutputStream(byteStream);
try {
genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, validRowIds.get(j),
- pageCounter, dataOutput);
+ .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, dimensionColumnPages,
+ validRowIds.get(j), pageCounter, reUseableDataOutput);
// get the key array in columnar way
byte[][] complexKeyArray = complexTypeArrayList.get(j);
complexKeyArray[i] = byteStream.toByteArray();
+ reUseableDataOutput.reset();
} catch (IOException e) {
+ isExceptionThrown = true;
LOGGER.error(e);
} finally {
- CarbonUtil.closeStreams(dataOutput);
- CarbonUtil.closeStreams(byteStream);
+ if (isExceptionThrown) {
+ CarbonUtil.closeStreams(reUseableDataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
}
}
}
+ CarbonUtil.closeStreams(reUseableDataOutput);
+ CarbonUtil.closeStreams(byteStream);
return complexTypeArrayList;
}
@@ -607,24 +623,32 @@ public abstract class BlockletScannedResult {
* @return complex type key array for all the complex dimension selected in query
*/
protected byte[][] getComplexTypeKeyArray(int rowId) {
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ ReUsableByteArrayDataOutputStream reUsableDataOutput =
+ new ReUsableByteArrayDataOutputStream(byteStream);
+ boolean isExceptionThrown = false;
byte[][] complexTypeData = new byte[complexParentBlockIndexes.length][];
for (int i = 0; i < complexTypeData.length; i++) {
GenericQueryType genericQueryType =
complexParentIndexToQueryMap.get(complexParentBlockIndexes[i]);
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutput = new DataOutputStream(byteStream);
try {
genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, rowId, pageCounter,
- dataOutput);
+ .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, dimensionColumnPages,
+ rowId, pageCounter, reUsableDataOutput);
complexTypeData[i] = byteStream.toByteArray();
+ reUsableDataOutput.reset();
} catch (IOException e) {
+ isExceptionThrown = true;
LOGGER.error(e);
} finally {
- CarbonUtil.closeStreams(dataOutput);
- CarbonUtil.closeStreams(byteStream);
+ if (isExceptionThrown) {
+ CarbonUtil.closeStreams(reUsableDataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
}
}
+ CarbonUtil.closeStreams(reUsableDataOutput);
+ CarbonUtil.closeStreams(byteStream);
return complexTypeData;
}
[07/21] carbondata git commit: [CARBONDATA-2838] Added SDV test cases
for Local Dictionary Support
Posted by ra...@apache.org.
[CARBONDATA-2838] Added SDV test cases for Local Dictionary Support
Added SDV test cases for Local Dictionary Support
This closes #2617
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ee8bfd05
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ee8bfd05
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ee8bfd05
Branch: refs/heads/branch-1.5
Commit: ee8bfd05f99d4bb26713021473d2547e1508c62f
Parents: 216d1c5
Author: praveenmeenakshi56 <pr...@gmail.com>
Authored: Wed Aug 8 11:45:49 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:33 2018 +0530
----------------------------------------------------------------------
.../sdv/generated/AlterTableTestCase.scala | 2 +-
.../LoadTableWithLocalDictionaryTestCase.scala | 325 +++++++++++++++++++
.../cluster/sdv/suite/SDVSuites.scala | 7 +-
3 files changed, 331 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee8bfd05/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala
index 90fa602..2cf1794 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/AlterTableTestCase.scala
@@ -1017,7 +1017,7 @@ class AlterTableTestCase extends QueryTest with BeforeAndAfterAll {
}
val prop = CarbonProperties.getInstance()
- val p1 = prop.getProperty("carbon.horizontal.compaction.enable", CarbonCommonConstants.defaultIsHorizontalCompactionEnabled)
+ val p1 = prop.getProperty("carbon.horizontal.compaction.enable", CarbonCommonConstants.CARBON_HORIZONTAL_COMPACTION_ENABLE_DEFAULT)
val p2 = prop.getProperty("carbon.horizontal.update.compaction.threshold", CarbonCommonConstants.DEFAULT_UPDATE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION)
val p3 = prop.getProperty("carbon.horizontal.delete.compaction.threshold", CarbonCommonConstants.DEFAULT_DELETE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION)
val p4 = prop.getProperty("carbon.compaction.level.threshold", CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee8bfd05/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LoadTableWithLocalDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LoadTableWithLocalDictionaryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LoadTableWithLocalDictionaryTestCase.scala
new file mode 100644
index 0000000..199358b
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LoadTableWithLocalDictionaryTestCase.scala
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.localdictionary
+
+import scala.collection.JavaConverters._
+import java.io.{File, PrintWriter}
+import java.util
+import java.util.Collections
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, Ignore}
+
+import org.apache.carbondata.core.cache.dictionary.DictionaryByteArrayWrapper
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.block.TableBlockInfo
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk
+import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
+import org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimensionChunkFileBasedReaderV3
+import org.apache.carbondata.core.datastore.compression.CompressorFactory
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.encoding.DefaultEncodingFactory
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion
+import org.apache.carbondata.core.util.{CarbonMetadataUtil, CarbonProperties, DataFileFooterConverterV3}
+
+class LoadTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file1 = resourcesPath + "/local_dictionary_source1.csv"
+
+ val file2 = resourcesPath + "/local_dictionary_complex_data.csv"
+
+ val storePath = warehouse + "/local2/Fact/Part0/Segment_0"
+
+ override protected def beforeAll(): Unit = {
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.BLOCKLET_SIZE, "10000")
+ createFile(file1)
+ createComplexData(file2)
+ sql("drop table if exists local2")
+ }
+
+ test("test LocalDictionary Load For FallBackScenario"){
+ sql("drop table if exists local2")
+ sql(
+ "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_threshold'='2001','local_dictionary_include'='name')")
+ sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+ assert(!checkForLocalDictionary(getDimRawChunk(0)))
+ }
+
+ test("test successful local dictionary generation"){
+ sql("drop table if exists local2")
+ sql(
+ "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_enable'='true','local_dictionary_threshold'='9001','local_dictionary_include'='name')")
+ sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+ assert(checkForLocalDictionary(getDimRawChunk(0)))
+ }
+
+ test("test successful local dictionary generation for default configs") {
+ sql("drop table if exists local2")
+ sql("CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')")
+ sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+ assert(checkForLocalDictionary(getDimRawChunk(0)))
+ }
+
+ test("test local dictionary generation for local dictioanry include") {
+ sql("drop table if exists local2")
+ sql(
+ "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+ "('dictionary_include'='name')")
+ sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+ assert(!checkForLocalDictionary(getDimRawChunk(0)))
+ }
+
+ test("test local dictionary generation for local dictioanry exclude"){
+ sql("drop table if exists local2")
+ sql(
+ "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_enable'='true','dictionary_exclude'='name')")
+ sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+ assert(checkForLocalDictionary(getDimRawChunk(0)))
+ }
+
+ test("test local dictionary generation when it is disabled"){
+ sql("drop table if exists local2")
+ sql(
+ "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_enable'='false','local_dictionary_include'='name')")
+ sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+ assert(!checkForLocalDictionary(getDimRawChunk(0)))
+ }
+
+ test("test local dictionary generation for invalid threshold configurations"){
+ sql("drop table if exists local2")
+ sql(
+ "CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_enable'='true','local_dictionary_include'='name','local_dictionary_threshold'='300000')")
+ sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
+ assert(checkForLocalDictionary(getDimRawChunk(0)))
+ }
+
+ test("test local dictionary generation for include and exclude"){
+ sql("drop table if exists local2")
+ sql(
+ "CREATE TABLE local2(name string, age string) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_enable'='true','local_dictionary_include'='name', 'local_dictionary_exclude'='age')")
+ sql("insert into table local2 values('vishal', '30')")
+ assert(checkForLocalDictionary(getDimRawChunk(0)))
+ assert(!checkForLocalDictionary(getDimRawChunk(1)))
+ }
+
+ test("test local dictionary generation for complex type"){
+ sql("drop table if exists local2")
+ sql(
+ "CREATE TABLE local2(name struct<i:string,s:string>) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_enable'='true','local_dictionary_include'='name')")
+ sql("load data inpath '" + file2 +
+ "' into table local2 OPTIONS('header'='false','COMPLEX_DELIMITER_LEVEL_1'='$', " +
+ "'COMPLEX_DELIMITER_LEVEL_2'=':')")
+ assert(!checkForLocalDictionary(getDimRawChunk(0)))
+ assert(checkForLocalDictionary(getDimRawChunk(1)))
+ assert(checkForLocalDictionary(getDimRawChunk(2)))
+ }
+
+ test("test local dictionary data validation") {
+ sql("drop table if exists local_query_enable")
+ sql("drop table if exists local_query_disable")
+ sql(
+ "CREATE TABLE local_query_enable(name string) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_enable'='false','local_dictionary_include'='name')")
+ sql("load data inpath '" + file1 + "' into table local_query_enable OPTIONS('header'='false')")
+ sql(
+ "CREATE TABLE local_query_disable(name string) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_enable'='true','local_dictionary_include'='name')")
+ sql("load data inpath '" + file1 + "' into table local_query_disable OPTIONS('header'='false')")
+ checkAnswer(sql("select name from local_query_enable"), sql("select name from local_query_disable"))
+ }
+
+ test("test to validate local dictionary values"){
+ sql("drop table if exists local2")
+ sql("CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')")
+ sql("load data inpath '" + resourcesPath + "/localdictionary.csv" + "' into table local2")
+ val dimRawChunk = getDimRawChunk(0)
+ val dictionaryData = Array("vishal", "kumar", "akash", "praveen", "brijoo")
+ try
+ assert(validateDictionary(dimRawChunk.get(0), dictionaryData))
+ catch {
+ case e: Exception =>
+ assert(false)
+ }
+ }
+
+ override protected def afterAll(): Unit = {
+ sql("drop table if exists local2")
+ deleteFile(file1)
+ deleteFile(file2)
+ CarbonProperties.getInstance
+ .addProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+ CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL)
+ }
+
+ /**
+ * create the csv data file
+ *
+ * @param fileName
+ */
+ private def createFile(fileName: String, line: Int = 9000, start: Int = 0): Unit = {
+ val writer = new PrintWriter(new File(fileName))
+ val data: util.ArrayList[String] = new util.ArrayList[String]()
+ for (i <- start until line) {
+ data.add("n" + i)
+ }
+ Collections.sort(data)
+ data.asScala.foreach { eachdata =>
+ writer.println(eachdata)
+ }
+ writer.close()
+ }
+
+ /**
+ * create the csv for complex data
+ *
+ * @param fileName
+ */
+ private def createComplexData(fileName: String, line: Int = 9000, start: Int = 0): Unit = {
+ val writer = new PrintWriter(new File(fileName))
+ val data: util.ArrayList[String] = new util.ArrayList[String]()
+ for (i <- start until line) {
+ data.add("n" + i + "$" + (i + 1))
+ }
+ Collections.sort(data)
+ data.asScala.foreach { eachdata =>
+ writer.println(eachdata)
+ }
+ writer.close()
+ }
+
+ /**
+ * delete csv file after test execution
+ *
+ * @param fileName
+ */
+ private def deleteFile(fileName: String): Unit = {
+ val file: File = new File(fileName)
+ if (file.exists) file.delete
+ }
+
+ private def checkForLocalDictionary(dimensionRawColumnChunks: util
+ .List[DimensionRawColumnChunk]): Boolean = {
+ var isLocalDictionaryGenerated = false
+ import scala.collection.JavaConversions._
+ for (dimensionRawColumnChunk <- dimensionRawColumnChunks) {
+ if (dimensionRawColumnChunk.getDataChunkV3
+ .isSetLocal_dictionary) {
+ isLocalDictionaryGenerated = true
+ }
+ }
+ isLocalDictionaryGenerated
+ }
+
+ /**
+ * this method returns true if local dictionary is created for all the blocklets or not
+ *
+ * @return
+ */
+ private def getDimRawChunk(blockindex: Int): util.ArrayList[DimensionRawColumnChunk] = {
+ val dataFiles = FileFactory.getCarbonFile(storePath)
+ .listFiles(new CarbonFileFilter() {
+ override def accept(file: CarbonFile): Boolean = {
+ if (file.getName
+ .endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
+ true
+ } else {
+ false
+ }
+ }
+ })
+ val dimensionRawColumnChunks = read(dataFiles(0).getAbsolutePath,
+ blockindex)
+ dimensionRawColumnChunks
+ }
+
+ private def read(filePath: String, blockIndex: Int) = {
+ val carbonDataFiles = new File(filePath)
+ val dimensionRawColumnChunks = new
+ util.ArrayList[DimensionRawColumnChunk]
+ val offset = carbonDataFiles.length
+ val converter = new DataFileFooterConverterV3
+ val fileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val actualOffset = fileReader.readLong(carbonDataFiles.getAbsolutePath, offset - 8)
+ val blockInfo = new TableBlockInfo(carbonDataFiles.getAbsolutePath,
+ actualOffset,
+ "0",
+ new Array[String](0),
+ carbonDataFiles.length,
+ ColumnarFormatVersion.V3,
+ null)
+ val dataFileFooter = converter.readDataFileFooter(blockInfo)
+ val blockletList = dataFileFooter.getBlockletList
+ import scala.collection.JavaConversions._
+ for (blockletInfo <- blockletList) {
+ val dimensionColumnChunkReader =
+ CarbonDataReaderFactory
+ .getInstance
+ .getDimensionColumnChunkReader(ColumnarFormatVersion.V3,
+ blockletInfo,
+ dataFileFooter.getSegmentInfo.getColumnCardinality,
+ carbonDataFiles.getAbsolutePath,
+ false).asInstanceOf[CompressedDimensionChunkFileBasedReaderV3]
+ dimensionRawColumnChunks
+ .add(dimensionColumnChunkReader.readRawDimensionChunk(fileReader, blockIndex))
+ }
+ dimensionRawColumnChunks
+ }
+
+ private def validateDictionary(rawColumnPage: DimensionRawColumnChunk,
+ data: Array[String]): Boolean = {
+ val local_dictionary = rawColumnPage.getDataChunkV3.local_dictionary
+ if (null != local_dictionary) {
+ val compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta(
+ rawColumnPage.getDataChunkV3.getData_chunk_list.get(0).getChunk_meta)
+ val encodings = local_dictionary.getDictionary_meta.encoders
+ val encoderMetas = local_dictionary.getDictionary_meta.getEncoder_meta
+ val encodingFactory = DefaultEncodingFactory.getInstance
+ val decoder = encodingFactory.createDecoder(encodings, encoderMetas, compressorName)
+ val dictionaryPage = decoder
+ .decode(local_dictionary.getDictionary_data, 0, local_dictionary.getDictionary_data.length)
+ val dictionaryMap = new
+ util.HashMap[DictionaryByteArrayWrapper, Integer]
+ val usedDictionaryValues = util.BitSet
+ .valueOf(CompressorFactory.getInstance.getCompressor(compressorName)
+ .unCompressByte(local_dictionary.getDictionary_values))
+ var index = 0
+ var i = usedDictionaryValues.nextSetBit(0)
+ while ( { i >= 0 }) {
+ dictionaryMap
+ .put(new DictionaryByteArrayWrapper(dictionaryPage.getBytes({ index += 1; index - 1 })),
+ i)
+ i = usedDictionaryValues.nextSetBit(i + 1)
+ }
+ for (i <- data.indices) {
+ if (null == dictionaryMap.get(new DictionaryByteArrayWrapper(data(i).getBytes))) {
+ return false
+ }
+ }
+ return true
+ }
+ false
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee8bfd05/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
index 6756468..3993042 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
@@ -21,6 +21,7 @@ import org.scalatest.{BeforeAndAfterAll, Suites}
import org.apache.carbondata.cluster.sdv.generated._
import org.apache.carbondata.cluster.sdv.register.TestRegisterCarbonTable
+import org.apache.carbondata.spark.testsuite.localdictionary.LoadTableWithLocalDictionaryTestCase
/**
* Suite class for all tests.
@@ -63,7 +64,8 @@ class SDVSuites extends Suites with BeforeAndAfterAll {
new TimeSeriesPreAggregateTestCase ::
new TestPartitionWithGlobalSort ::
new PartitionWithPreAggregateTestCase ::
- new CreateTableWithLocalDictionaryTestCase :: Nil
+ new CreateTableWithLocalDictionaryTestCase ::
+ new LoadTableWithLocalDictionaryTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
@@ -154,7 +156,8 @@ class SDVSuites3 extends Suites with BeforeAndAfterAll {
new SDKwriterTestCase ::
new SetParameterTestCase ::
new PartitionWithPreAggregateTestCase ::
- new CreateTableWithLocalDictionaryTestCase :: Nil
+ new CreateTableWithLocalDictionaryTestCase ::
+ new LoadTableWithLocalDictionaryTestCase :: Nil
override val nestedSuites = suites.toIndexedSeq
[19/21] carbondata git commit: [CARBONDATA-3166]Updated Document and
added Column Compressor used in Describe Formatted
Posted by ra...@apache.org.
[CARBONDATA-3166]Updated Document and added Column Compressor used in Describe Formatted
Updated Document and added column compressor used in Describe Formatted Command
This closes #2986
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/89351649
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/89351649
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/89351649
Branch: refs/heads/branch-1.5
Commit: 893516498c00d287ddb97b9faf04821dbd36c6d7
Parents: 3c59721
Author: shardul-cr7 <sh...@gmail.com>
Authored: Thu Dec 13 14:12:18 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:34 2018 +0530
----------------------------------------------------------------------
docs/configuration-parameters.md | 2 +-
.../execution/command/table/CarbonDescribeFormattedCommand.scala | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89351649/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 4aa2929..db21c6a 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -91,7 +91,7 @@ This section provides the details of all the configurations required for the Car
| carbon.dictionary.server.port | 2030 | Single Pass Loading enables single job to finish data loading with dictionary generation on the fly. It enhances performance in the scenarios where the subsequent data loading after initial load involves fewer incremental updates on the dictionary. Single pass loading can be enabled using the option ***carbon.options.single.pass***. When this option is specified, a dictionary server will be internally started to handle the dictionary generation and query requests. This configuration specifies the port on which the server need to listen for incoming requests. Port value ranges between 0-65535 |
| carbon.load.directWriteToStorePath.enabled | false | During data load, all the carbondata files are written to local disk and finally copied to the target store location in HDFS/S3. Enabling this parameter will make carbondata files to be written directly onto target HDFS/S3 location bypassing the local disk.**NOTE:** Writing directly to HDFS/S3 saves local disk IO(once for writing the files and again for copying to HDFS/S3) there by improving the performance. But the drawback is when data loading fails or the application crashes, unwanted carbondata files will remain in the target HDFS/S3 location until it is cleared during next data load or by running *CLEAN FILES* DDL command |
| carbon.options.serialization.null.format | \N | Based on the business scenarios, some columns might need to be loaded with null values. As null value cannot be written in csv files, some special characters might be adopted to specify null values. This configuration can be used to specify the null values format in the data being loaded. |
-| carbon.column.compressor | snappy | CarbonData will compress the column values using the compressor specified by this configuration. Currently CarbonData supports 'snappy' and 'zstd' compressors. |
+| carbon.column.compressor | snappy | CarbonData will compress the column values using the compressor specified by this configuration. Currently CarbonData supports 'snappy', 'zstd' and 'gzip' compressors. |
| carbon.minmax.allowed.byte.count | 200 | CarbonData will write the min max values for string/varchar types column using the byte count specified by this configuration. Max value is 1000 bytes(500 characters) and Min value is 10 bytes(5 characters). **NOTE:** This property is useful for reducing the store size thereby improving the query performance but can lead to query degradation if value is not configured properly. | |
## Compaction Configuration
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89351649/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index 151359e..2d560df 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -92,7 +92,9 @@ private[sql] case class CarbonDescribeFormattedCommand(
Strings.formatSize(
tblProps.getOrElse(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB,
CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB_DEFAULT).toFloat), ""),
-
+ ("Data File Compressor ", tblProps
+ .getOrElse(CarbonCommonConstants.COMPRESSOR,
+ CarbonCommonConstants.DEFAULT_COMPRESSOR), ""),
//////////////////////////////////////////////////////////////////////////////
// Index Information
//////////////////////////////////////////////////////////////////////////////
[14/21] carbondata git commit: [CARBONDATA-3017] Map DDL Support
Posted by ra...@apache.org.
[CARBONDATA-3017] Map DDL Support
Support Create DDL for Map type.
This closes #2980
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c9b136c2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c9b136c2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c9b136c2
Branch: refs/heads/branch-1.5
Commit: c9b136c26763baad9bb8f81aa3676217578d6ccc
Parents: 8935164
Author: manishnalla1994 <ma...@gmail.com>
Authored: Tue Oct 16 15:18:08 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:34 2018 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 4 +
.../hadoop/api/CarbonTableOutputFormat.java | 17 +-
.../TestCreateDDLForComplexMapType.scala | 445 +++++++++++++++++++
.../LocalDictionarySupportLoadTableTest.scala | 17 +
.../spark/rdd/NewCarbonDataLoadRDD.scala | 10 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 1 +
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 43 +-
.../streaming/CarbonAppendableStreamSink.scala | 9 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 4 +-
.../CarbonAlterTableCompactionCommand.scala | 6 +
.../management/CarbonLoadDataCommand.scala | 6 +-
.../table/CarbonCreateTableCommand.scala | 2 +-
.../spark/util/AllDictionaryTestCase.scala | 4 +-
.../util/ExternalColumnDictionaryTestCase.scala | 4 +-
.../TestStreamingTableWithRowParser.scala | 3 +-
.../loading/ComplexDelimitersEnum.java | 39 ++
.../loading/DataLoadProcessBuilder.java | 7 +-
.../loading/model/CarbonLoadModel.java | 38 +-
.../loading/model/CarbonLoadModelBuilder.java | 11 +-
.../processing/loading/model/LoadOption.java | 17 +-
.../loading/parser/CarbonParserFactory.java | 25 +-
.../loading/parser/impl/ArrayParserImpl.java | 6 +-
.../loading/parser/impl/MapParserImpl.java | 60 +++
.../loading/parser/impl/RowParserImpl.java | 8 +-
.../sdk/file/CarbonWriterBuilder.java | 1 +
.../streaming/parser/RowStreamParserImp.scala | 2 +
26 files changed, 715 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index ac52728..fc4704e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -597,6 +597,10 @@ public final class CarbonUtil {
*/
public static String delimiterConverter(String delimiter) {
switch (delimiter) {
+ case "\\001":
+ case "\\002":
+ case "\\003":
+ case "\\004":
case "|":
case "*":
case ".":
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index dbd2f0e..16486d0 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.ObjectSerializationUtil;
import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
+import org.apache.carbondata.processing.loading.ComplexDelimitersEnum;
import org.apache.carbondata.processing.loading.DataLoadExecutor;
import org.apache.carbondata.processing.loading.TableProcessingOperations;
import org.apache.carbondata.processing.loading.iterator.CarbonOutputIteratorWrapper;
@@ -338,11 +339,19 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Obje
SKIP_EMPTY_LINE,
carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
- String complexDelim = conf.get(COMPLEX_DELIMITERS, "\\\001" + "," + "\\\002");
+ String complexDelim = conf.get(COMPLEX_DELIMITERS);
+ if (null == complexDelim) {
+ complexDelim = ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_1.value() + ","
+ + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_2.value() + ","
+ + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_3.value();
+ }
String[] split = complexDelim.split(",");
- model.setComplexDelimiterLevel1(split[0]);
- if (split.length > 1) {
- model.setComplexDelimiterLevel2(split[1]);
+ model.setComplexDelimiter(split[0]);
+ if (split.length > 2) {
+ model.setComplexDelimiter(split[1]);
+ model.setComplexDelimiter(split[2]);
+ } else if (split.length > 1) {
+ model.setComplexDelimiter(split[1]);
}
model.setDateFormat(
conf.get(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateDDLForComplexMapType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateDDLForComplexMapType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateDDLForComplexMapType.scala
new file mode 100644
index 0000000..941364c
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateDDLForComplexMapType.scala
@@ -0,0 +1,445 @@
+/*
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ *
+ http://www.apache.org/licenses/LICENSE-2.0
+ *
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.createTable.TestCreateDDLForComplexMapType
+
+import java.io.{BufferedWriter, File, FileWriter}
+import java.util
+
+import au.com.bytecode.opencsv.CSVWriter
+import org.apache.hadoop.conf.Configuration
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk
+
+import scala.collection.JavaConversions._
+
+class TestCreateDDLForComplexMapType extends QueryTest with BeforeAndAfterAll {
+ private val conf: Configuration = new Configuration(false)
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+
+ val path = s"$rootPath/integration/spark-common-test/src/test/resources/maptest2.csv"
+
+ private def checkForLocalDictionary(dimensionRawColumnChunks: util
+ .List[DimensionRawColumnChunk]): Boolean = {
+ var isLocalDictionaryGenerated = false
+ import scala.collection.JavaConversions._
+ isLocalDictionaryGenerated = dimensionRawColumnChunks
+ .filter(dimensionRawColumnChunk => dimensionRawColumnChunk.getDataChunkV3
+ .isSetLocal_dictionary).size > 0
+ isLocalDictionaryGenerated
+ }
+
+ def createCSVFile(): Unit = {
+ val out = new BufferedWriter(new FileWriter(path));
+ val writer = new CSVWriter(out);
+
+ val employee1 = Array("1\u0002Nalla\u00012\u0002Singh\u00011\u0002Gupta\u00014\u0002Kumar")
+
+ val employee2 = Array("10\u0002Nallaa\u000120\u0002Sissngh\u0001100\u0002Gusspta\u000140" +
+ "\u0002Kumar")
+
+ var listOfRecords = List(employee1, employee2)
+
+ writer.writeAll(listOfRecords)
+ out.close()
+ }
+
+ override def beforeAll(): Unit = {
+ createCSVFile()
+ sql("DROP TABLE IF EXISTS carbon")
+ }
+
+ override def afterAll(): Unit = {
+ new File(path).delete()
+ }
+
+ test("Single Map One Level") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<STRING,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ val desc = sql(
+ s"""
+ | Describe Formatted
+ | carbon
+ | """.stripMargin).collect()
+ assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,string>"))
+ }
+
+ test("Single Map with Two Nested Level") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<STRING,map<INT,STRING>>
+ | )
+ | STORED BY
+ |'carbondata'
+ |"""
+ .stripMargin)
+ val desc = sql(
+ s"""
+ | Describe Formatted
+ | carbon
+ | """.stripMargin).collect()
+ assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,map<int,string>>"))
+ }
+
+ test("Map Type with array type as value") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<STRING,array<INT>>
+ | )
+ | STORED BY 'carbondata'
+ |
+ """
+ .stripMargin)
+ val desc = sql(
+ s"""
+ | Describe Formatted
+ | carbon
+ | """.stripMargin).collect()
+ assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,array<int>>"))
+ }
+
+ test("Map Type with struct type as value") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<STRING,struct<key:INT,val:INT>>
+ | )
+ | STORED BY
+ | 'carbondata'
+ | """
+ .stripMargin)
+ val desc = sql(
+ s"""
+ | Describe Formatted
+ | carbon
+ | """.stripMargin).collect()
+ assert(desc(0).get(1).asInstanceOf[String].trim
+ .equals("map<string,struct<key:int,val:int>>"))
+ }
+
+ test("Map Type as child to struct type") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField struct<key:INT,val:map<INT,INT>>
+ | )
+ | STORED BY
+ |'carbondata' """
+ .stripMargin)
+ val desc = sql(
+ s"""
+ | Describe Formatted
+ | carbon
+ | """.stripMargin).collect()
+ assert(desc(0).get(1).asInstanceOf[String].trim
+ .equals("struct<key:int,val:map<int,int>>"))
+ }
+
+ test("Map Type as child to array type") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField array<map<INT,INT>>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ val desc = sql(
+ s"""
+ | Describe Formatted
+ | carbon """.stripMargin).collect()
+ assert(desc(0).get(1).asInstanceOf[String].trim.equals("array<map<int,int>>"))
+ sql("insert into carbon values('1\0032\0022\0033\001100\003200\002200\003300')")
+ sql("select * from carbon").show(false)
+ }
+
+ test("Test Load data in map") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<INT,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ val desc = sql(
+ s"""
+ | Describe Formatted
+ | carbon
+ | """.stripMargin).collect()
+ sql("insert into carbon values('1\002Nalla\0012\002Singh\0013\002Gupta\0014\002Kumar')")
+ checkAnswer(sql("select * from carbon"), Seq(
+ Row(Map(1 -> "Nalla", 2 -> "Singh", 3 -> "Gupta", 4 -> "Kumar"))))
+ }
+
+ test("Test Load data in map with empty value") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<INT,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ val desc = sql(
+ s"""
+ | Describe Formatted
+ | carbon
+ | """.stripMargin).collect()
+ sql("insert into carbon values('1\002Nalla\0012\002\0013\002Gupta\0014\002Kumar')")
+ checkAnswer(sql("select * from carbon"), Seq(
+ Row(Map(1 -> "Nalla", 2 -> "", 3 -> "Gupta", 4 -> "Kumar"))))
+ }
+
+ // Support this for Map type
+ test("Test Load data in map with dictionary include") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<int,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('DICTIONARY_INCLUDE'='mapField')
+ | """
+ .stripMargin)
+ sql("insert into carbon values('1\002Nalla\0012\002Singh\0013\002Gupta')")
+ sql("select * from carbon").show(false)
+ //checkAnswer(sql("select * from carbon"), Seq(
+ //Row(Map(1 -> "Nalla", 2 -> "Singh", 3 -> "Gupta", 4 -> "Kumar"))))
+ }
+
+ test("Test Load data in map with partition columns") {
+ sql("DROP TABLE IF EXISTS carbon")
+ val exception = intercept[AnalysisException](
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | a INT,
+ | mapField array<STRING>,
+ | b STRING
+ | )
+ | PARTITIONED BY (mp map<int,string>)
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ )
+ assertResult("Cannot use map<int,string> for partition column;")(exception.getMessage())
+ }
+
+ test("Test IUD in map columns") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | a INT,
+ | mapField map<INT,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ sql("insert into carbon values(1,'1\002Nalla\0012\002Singh\0013\002Gupta\0014\002Kumar')")
+ sql("insert into carbon values(2,'1\002abc\0012\002xyz\0013\002hello\0014\002mno')")
+ val exception = intercept[UnsupportedOperationException](
+ sql("update carbon set(mapField)=('1,haha') where a=1").show(false))
+ assertResult("Unsupported operation on Complex data type")(exception.getMessage())
+ sql("delete from carbon where mapField[1]='abc'")
+ checkAnswer(sql("select * from carbon"), Seq(
+ Row(1, Map(1 -> "Nalla", 2 -> "Singh", 3 -> "Gupta", 4 -> "Kumar"))))
+
+ }
+
+ test("Test Compaction blocking") {
+ sql("DROP TABLE IF EXISTS carbon")
+
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | a INT,
+ | mapField map<INT,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+
+ val exception = intercept[UnsupportedOperationException](
+ sql("ALTER table carbon compact 'minor'")
+ )
+ assertResult("Compaction is unsupported for Table containing Map Columns")(exception
+ .getMessage())
+ }
+
+ test("Test Load duplicate keys data in map") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<INT,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ val desc = sql(
+ s"""
+ | Describe Formatted
+ | carbon
+ | """.stripMargin).collect()
+ sql("insert into carbon values('1\002Nalla\0012\002Singh\0011\002Gupta\0014\002Kumar')")
+ checkAnswer(sql("select * from carbon"), Seq(
+ Row(Map(1 -> "Nalla", 2 -> "Singh", 4 -> "Kumar"))))
+ }
+
+ test("Test Load data in map of map") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<STRING,map<INT,STRING>>
+ | )
+ | STORED BY
+ |'carbondata' """
+ .stripMargin)
+ sql(
+ "insert into carbon values('manish\0021\004nalla\0032\004gupta\001kunal\0021\004kapoor\0032" +
+ "\004sharma')")
+ checkAnswer(sql("select * from carbon"), Seq(
+ Row(Map("manish" -> Map(1 -> "nalla", 2 -> "gupta"),
+ "kunal" -> Map(1 -> "kapoor", 2 -> "sharma")))))
+ }
+
+ test("Test Load duplicate keys data in map of map") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<STRING,map<INT,STRING>>
+ | )
+ | STORED BY
+ |'carbondata'
+ |"""
+ .stripMargin)
+ sql(
+ "insert into carbon values('manish\0021\004nalla\0031\004gupta\001kunal\0021\004kapoor\0032" +
+ "\004sharma')")
+ checkAnswer(sql("select * from carbon"), Seq(
+ Row(Map("manish" -> Map(1 -> "nalla"),
+ "kunal" -> Map(1 -> "kapoor", 2 -> "sharma")))))
+ }
+
+ test("Test Create table as select with map") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql("DROP TABLE IF EXISTS carbon1")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<INT,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ sql("insert into carbon values('1\002Nalla\0012\002Singh\0013\002Gupta\0014\002Kumar')")
+ sql(
+ s"""
+ | CREATE TABLE carbon1
+ | AS
+ | Select *
+ | From carbon
+ | """
+ .stripMargin)
+ checkAnswer(sql("select * from carbon1"), Seq(
+ Row(Map(1 -> "Nalla", 2 -> "Singh", 3 -> "Gupta", 4 -> "Kumar"))))
+ }
+
+ test("Test Create table with double datatype in map") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<DOUBLE,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ sql(
+ "insert into carbon values('1.23\002Nalla\0012.34\002Singh\0013.67676\002Gupta\0013.67676" +
+ "\002Kumar')")
+ checkAnswer(sql("select * from carbon"), Seq(
+ Row(Map(1.23 -> "Nalla", 2.34 -> "Singh", 3.67676 -> "Gupta"))))
+ }
+
+ test("Load Map data from CSV File") {
+ sql("DROP TABLE IF EXISTS carbon")
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<INT,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | """
+ .stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon OPTIONS(
+ | 'header' = 'false')
+ """.stripMargin)
+ checkAnswer(sql("select * from carbon"), Seq(
+ Row(Map(1 -> "Nalla", 2 -> "Singh", 4 -> "Kumar")),
+ Row(Map(10 -> "Nallaa", 20 -> "Sissngh", 100 -> "Gusspta", 40 -> "Kumar"))
+ ))
+ }
+
+ test("Sort Column table property blocking for Map type") {
+ sql("DROP TABLE IF EXISTS carbon")
+ val exception1 = intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE carbon(
+ | mapField map<STRING,STRING>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('SORT_COLUMNS'='mapField')
+ | """
+ .stripMargin)
+ }
+ assert(exception1.getMessage
+ .contains(
+ "sort_columns is unsupported for map datatype column: mapfield"))
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
index d23c844..d332a5a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
@@ -21,6 +21,7 @@ import java.io.{File, PrintWriter}
import java.util
import java.util.Collections
+import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, Ignore}
@@ -136,6 +137,22 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA
assert(checkForLocalDictionary(getDimRawChunk(2)))
}
+ test("test local dictionary generation for map type") {
+ sql("drop table if exists local2")
+ sql(
+ "CREATE TABLE local2(name map<string,string>) STORED BY 'carbondata' tblproperties" +
+ "('local_dictionary_enable'='true','local_dictionary_include'='name')")
+ sql(
+ "insert into local2 values('Manish\002Nalla\001Manish\002Gupta\001Shardul\002Singh" +
+ "\001Vishal\002Kumar')")
+ checkAnswer(sql("select * from local2"), Seq(
+ Row(Map("Manish" -> "Nalla", "Shardul" -> "Singh", "Vishal" -> "Kumar"))))
+ assert(!checkForLocalDictionary(getDimRawChunk(0)))
+ assert(!checkForLocalDictionary(getDimRawChunk(1)))
+ assert(checkForLocalDictionary(getDimRawChunk(2)))
+ assert(checkForLocalDictionary(getDimRawChunk(3)))
+ }
+
test("test local dictionary data validation") {
sql("drop table if exists local_query_enable")
sql("drop table if exists local_query_disable")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 3752eef..f7249b8 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -339,8 +339,9 @@ class NewRddIterator(rddIter: Iterator[Row],
private val dateFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
private val dateFormat = new SimpleDateFormat(dateFormatString)
- private val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
- private val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
+ private val delimiterLevel1 = carbonLoadModel.getComplexDelimiters.get(0)
+ private val delimiterLevel2 = carbonLoadModel.getComplexDelimiters.get(1)
+ private val delimiterLevel3 = carbonLoadModel.getComplexDelimiters.get(2)
private val serializationNullFormat =
carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
import scala.collection.JavaConverters._
@@ -388,8 +389,9 @@ class LazyRddIterator(serializer: SerializerInstance,
.getProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
private val dateFormat = new SimpleDateFormat(dateFormatString)
- private val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
- private val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
+ private val delimiterLevel1 = carbonLoadModel.getComplexDelimiters.get(0)
+ private val delimiterLevel2 = carbonLoadModel.getComplexDelimiters.get(1)
+ private val delimiterLevel3 = carbonLoadModel.getComplexDelimiters.get(2)
private val serializationNullFormat =
carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
// the order of fields in dataframe and createTable may be different, here we need to know whether
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 71447e9..ca9b4af 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -643,6 +643,7 @@ object CarbonScalaUtil {
!x.dataType.get.equalsIgnoreCase("STRING") &&
!x.dataType.get.equalsIgnoreCase("VARCHAR") &&
!x.dataType.get.equalsIgnoreCase("STRUCT") &&
+ !x.dataType.get.equalsIgnoreCase("MAP") &&
!x.dataType.get.equalsIgnoreCase("ARRAY"))) {
val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
dictColm.trim +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 3ac2d2b..c971573 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -114,6 +114,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
protected val MULTILINE = carbonKeyWord("MULTILINE")
protected val COMPLEX_DELIMITER_LEVEL_1 = carbonKeyWord("COMPLEX_DELIMITER_LEVEL_1")
protected val COMPLEX_DELIMITER_LEVEL_2 = carbonKeyWord("COMPLEX_DELIMITER_LEVEL_2")
+ protected val COMPLEX_DELIMITER_LEVEL_3 = carbonKeyWord("COMPLEX_DELIMITER_LEVEL_3")
protected val OPTIONS = carbonKeyWord("OPTIONS")
protected val OUTPATH = carbonKeyWord("OUTPATH")
protected val OVERWRITE = carbonKeyWord("OVERWRITE")
@@ -915,7 +916,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
* @param dimensionDatatype
*/
def isDetectAsDimentionDatatype(dimensionDatatype: String): Boolean = {
- val dimensionType = Array("string", "array", "struct", "timestamp", "date", "char")
+ val dimensionType = Array("string", "array", "struct", "map", "timestamp", "date", "char")
dimensionType.exists(x => dimensionDatatype.toLowerCase.contains(x))
}
@@ -1070,13 +1071,32 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
// validate with all supported options
val options = optionList.get.groupBy(x => x._1)
- val supportedOptions = Seq("DELIMITER", "QUOTECHAR", "FILEHEADER", "ESCAPECHAR", "MULTILINE",
- "COMPLEX_DELIMITER_LEVEL_1", "COMPLEX_DELIMITER_LEVEL_2", "COLUMNDICT",
- "SERIALIZATION_NULL_FORMAT", "BAD_RECORDS_LOGGER_ENABLE", "BAD_RECORDS_ACTION",
- "ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "DATEFORMAT", "BAD_RECORD_PATH",
- "BATCH_SORT_SIZE_INMB", "GLOBAL_SORT_PARTITIONS", "SINGLE_PASS",
- "IS_EMPTY_DATA_BAD_RECORD", "HEADER", "TIMESTAMPFORMAT", "SKIP_EMPTY_LINE",
- "SORT_COLUMN_BOUNDS", "LOAD_MIN_SIZE_INMB"
+ val supportedOptions = Seq("DELIMITER",
+ "QUOTECHAR",
+ "FILEHEADER",
+ "ESCAPECHAR",
+ "MULTILINE",
+ "COMPLEX_DELIMITER_LEVEL_1",
+ "COMPLEX_DELIMITER_LEVEL_2",
+ "COMPLEX_DELIMITER_LEVEL_3",
+ "COLUMNDICT",
+ "SERIALIZATION_NULL_FORMAT",
+ "BAD_RECORDS_LOGGER_ENABLE",
+ "BAD_RECORDS_ACTION",
+ "ALL_DICTIONARY_PATH",
+ "MAXCOLUMNS",
+ "COMMENTCHAR",
+ "DATEFORMAT",
+ "BAD_RECORD_PATH",
+ "BATCH_SORT_SIZE_INMB",
+ "GLOBAL_SORT_PARTITIONS",
+ "SINGLE_PASS",
+ "IS_EMPTY_DATA_BAD_RECORD",
+ "HEADER",
+ "TIMESTAMPFORMAT",
+ "SKIP_EMPTY_LINE",
+ "SORT_COLUMN_BOUNDS",
+ "LOAD_MIN_SIZE_INMB"
)
var isSupported = true
val invalidOptions = StringBuilder.newBuilder
@@ -1291,13 +1311,16 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
Field("unknown", Some("struct"), Some("unknown"), Some(e1))
}
+ // Map<Key,Value> is represented as Map<Struct<Key,Value>>
protected lazy val mapFieldType: Parser[Field] =
(MAP ^^^ "map") ~> "<" ~> primitiveFieldType ~ ("," ~> nestedType) <~ ">" ^^ {
case key ~ value =>
Field("unknown", Some("map"), Some("unknown"),
Some(List(
- Field("key", key.dataType, Some("key"), key.children),
- Field("value", value.dataType, Some("value"), value.children))))
+ Field("val", Some("struct"), Some("unknown"),
+ Some(List(
+ Field("key", key.dataType, Some("key"), key.children),
+ Field("value", value.dataType, Some("value"), value.children)))))))
}
protected lazy val measureCol: Parser[Field] =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index 3d8170e..184cc1d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.execution.streaming
+import java.util
import java.util.Date
import scala.collection.JavaConverters._
@@ -77,10 +78,10 @@ class CarbonAppendableStreamSink(
conf.set(entry._1, entry._2)
}
// properties below will be used for default CarbonStreamParser
- conf.set("carbon_complex_delimiter_level_1",
- carbonLoadModel.getComplexDelimiterLevel1)
- conf.set("carbon_complex_delimiter_level_2",
- carbonLoadModel.getComplexDelimiterLevel2)
+ val complexDelimiters = carbonLoadModel.getComplexDelimiters
+ conf.set("carbon_complex_delimiter_level_1", complexDelimiters.get(0))
+ conf.set("carbon_complex_delimiter_level_2", complexDelimiters.get(1))
+ conf.set("carbon_complex_delimiter_level_3", complexDelimiters.get(2))
conf.set(
DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT,
carbonLoadModel.getSerializationNullFormat().split(",")(1))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index b07be72..a849c99 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -989,8 +989,8 @@ object CarbonDataRDDFactory {
// generate RDD[(K, V)] to use the partitionBy method of PairRDDFunctions
val inputRDD: RDD[(String, Row)] = if (dataFrame.isDefined) {
// input data from DataFrame
- val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
- val delimiterLevel2 = carbonLoadModel.getComplexDelimiterLevel2
+ val delimiterLevel1 = carbonLoadModel.getComplexDelimiters.get(0)
+ val delimiterLevel2 = carbonLoadModel.getComplexDelimiters.get(1)
val serializationNullFormat =
carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
dataFrame.get.rdd.map { row =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index f98c0cf..6edfcf4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -39,6 +39,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.metadata.ColumnarFormatVersion
+import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
@@ -87,6 +88,11 @@ case class CarbonAlterTableCompactionCommand(
if (!table.getTableInfo.isTransactionalTable) {
throw new MalformedCarbonCommandException("Unsupported operation on non transactional table")
}
+ if (table.getTableInfo.getFactTable.getListOfColumns.asScala
+ .exists((m => DataTypes.isMapType(m.getDataType)))) {
+ throw new UnsupportedOperationException(
+ "Compaction is unsupported for Table containing Map Columns")
+ }
if (CarbonUtil.hasAggregationDataMap(table) ||
(table.isChildDataMap && null == operationContext.getProperty(table.getTableName))) {
// If the compaction request is of 'streaming' type then we need to generate loadCommands
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 6a8a9cf..7a974e3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -69,7 +69,7 @@ import org.apache.carbondata.core.util._
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{BuildDataMapPostExecutionEvent, BuildDataMapPreExecutionEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.events.exception.PreEventException
-import org.apache.carbondata.processing.loading.TableProcessingOperations
+import org.apache.carbondata.processing.loading.{ComplexDelimitersEnum, TableProcessingOperations}
import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadMetadataEvent, LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
import org.apache.carbondata.processing.loading.exception.NoRetryException
import org.apache.carbondata.processing.loading.model.{CarbonLoadModelBuilder, LoadOption}
@@ -188,11 +188,13 @@ case class CarbonLoadDataCommand(
val carbonLoadModel = new CarbonLoadModel()
val tableProperties = table.getTableInfo.getFactTable.getTableProperties
val optionsFinal = LoadOption.fillOptionWithDefaultValue(options.asJava)
+ optionsFinal
+ .put("complex_delimiter_level_4",
+ ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_4.value())
optionsFinal.put("sort_scope", tableProperties.asScala.getOrElse("sort_scope",
carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
carbonProperty.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))))
-
optionsFinal
.put("bad_record_path", CarbonBadRecordUtil.getBadRecordsPath(options.asJava, table))
val factPath = if (dataFrame.isDefined) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index ca39931..713561b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -128,7 +128,7 @@ case class CarbonCreateTableCommand(
if (partitionInfo != null &&
partitionInfo.getPartitionType == PartitionType.NATIVE_HIVE) {
// Restrict dictionary encoding on partition columns.
- // TODO Need to decide wherher it is required
+ // TODO Need to decide whether it is required
val dictionaryOnPartitionColumn =
partitionInfo.getColumnSchemaList.asScala.exists{p =>
p.hasEncoding(Encoding.DICTIONARY) && !p.hasEncoding(Encoding.DIRECT_DICTIONARY)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index 08c149b..ed5486b 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -55,8 +55,8 @@ class AllDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll {
carbonLoadModel.setFactFilePath(filePath)
carbonLoadModel.setCsvHeader(header)
carbonLoadModel.setCsvDelimiter(",")
- carbonLoadModel.setComplexDelimiterLevel1("$")
- carbonLoadModel.setComplexDelimiterLevel2(":")
+ carbonLoadModel.setComplexDelimiter("$")
+ carbonLoadModel.setComplexDelimiter(":")
carbonLoadModel.setAllDictPath(allDictFilePath)
carbonLoadModel.setSerializationNullFormat(
TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + ",\\N")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index 060afca..69248d6 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -165,8 +165,8 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
carbonLoadModel.setFactFilePath(filePath)
carbonLoadModel.setCsvHeader(header)
carbonLoadModel.setCsvDelimiter(csvDelimiter)
- carbonLoadModel.setComplexDelimiterLevel1("$")
- carbonLoadModel.setComplexDelimiterLevel2(":")
+ carbonLoadModel.setComplexDelimiter("$")
+ carbonLoadModel.setComplexDelimiter(":")
carbonLoadModel.setColDictFilePath(extColFilePath)
carbonLoadModel.setQuoteChar("\"");
carbonLoadModel.setSerializationNullFormat(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
index 985b9d9..21cad07 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
@@ -42,7 +42,6 @@ case class StreamData(id: Integer, name: String, city: String, salary: java.lang
register: String, updated: String,
file: FileElement)
-@Ignore
class TestStreamingTableWithRowParser extends QueryTest with BeforeAndAfterAll {
private val spark = sqlContext.sparkSession
@@ -420,7 +419,7 @@ class TestStreamingTableWithRowParser extends QueryTest with BeforeAndAfterAll {
continueSeconds = 20,
generateBadRecords = true,
badRecordAction = "force",
- autoHandoff = true
+ autoHandoff = false
)
// non-filter
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/processing/src/main/java/org/apache/carbondata/processing/loading/ComplexDelimitersEnum.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/ComplexDelimitersEnum.java b/processing/src/main/java/org/apache/carbondata/processing/loading/ComplexDelimitersEnum.java
new file mode 100644
index 0000000..bc196e1
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/ComplexDelimitersEnum.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading;
+
+public enum ComplexDelimitersEnum {
+
+ COMPLEX_DELIMITERS_LEVEL_1("\001"),
+
+ COMPLEX_DELIMITERS_LEVEL_2("\002"),
+
+ COMPLEX_DELIMITERS_LEVEL_3("\003"),
+
+ COMPLEX_DELIMITERS_LEVEL_4("\004");
+
+ private String value;
+
+ ComplexDelimitersEnum(String value) {
+ this.value = value;
+ }
+
+ public String value() {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index 89d09fe..6fe89a2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -221,9 +221,10 @@ public final class DataLoadProcessBuilder {
configuration.setHeader(loadModel.getCsvHeaderColumns());
configuration.setSegmentId(loadModel.getSegmentId());
configuration.setTaskNo(loadModel.getTaskNo());
- configuration.setDataLoadProperty(DataLoadProcessorConstants.COMPLEX_DELIMITERS,
- new String[] { loadModel.getComplexDelimiterLevel1(),
- loadModel.getComplexDelimiterLevel2() });
+ String[] complexDelimiters = new String[loadModel.getComplexDelimiters().size()];
+ loadModel.getComplexDelimiters().toArray(complexDelimiters);
+ configuration
+ .setDataLoadProperty(DataLoadProcessorConstants.COMPLEX_DELIMITERS, complexDelimiters);
configuration.setDataLoadProperty(DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT,
loadModel.getSerializationNullFormat().split(",")[1]);
configuration.setDataLoadProperty(DataLoadProcessorConstants.FACT_TIME_STAMP,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index e15fb5d..aecc52e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -65,8 +65,7 @@ public class CarbonLoadModel implements Serializable {
private String csvHeader;
private String[] csvHeaderColumns;
private String csvDelimiter;
- private String complexDelimiterLevel1;
- private String complexDelimiterLevel2;
+ private ArrayList<String> complexDelimiters;
private List<LoadMetadataDetails> loadMetadataDetails;
private transient SegmentUpdateStatusManager segmentUpdateStatusManager;
@@ -276,20 +275,14 @@ public class CarbonLoadModel implements Serializable {
this.csvDelimiter = csvDelimiter;
}
- public String getComplexDelimiterLevel1() {
- return complexDelimiterLevel1;
+ public void setComplexDelimiter(String delimiter) {
+ checkAndInitializeComplexDelimiterList();
+ this.complexDelimiters.add(delimiter);
}
- public void setComplexDelimiterLevel1(String complexDelimiterLevel1) {
- this.complexDelimiterLevel1 = complexDelimiterLevel1;
- }
-
- public String getComplexDelimiterLevel2() {
- return complexDelimiterLevel2;
- }
-
- public void setComplexDelimiterLevel2(String complexDelimiterLevel2) {
- this.complexDelimiterLevel2 = complexDelimiterLevel2;
+ public ArrayList<String> getComplexDelimiters() {
+ checkAndInitializeComplexDelimiterList();
+ return complexDelimiters;
}
public String getAllDictPath() {
@@ -441,8 +434,7 @@ public class CarbonLoadModel implements Serializable {
copy.csvHeader = csvHeader;
copy.csvHeaderColumns = csvHeaderColumns;
copy.csvDelimiter = csvDelimiter;
- copy.complexDelimiterLevel1 = complexDelimiterLevel1;
- copy.complexDelimiterLevel2 = complexDelimiterLevel2;
+ copy.complexDelimiters = complexDelimiters;
copy.carbonDataLoadSchema = carbonDataLoadSchema;
copy.blocksID = blocksID;
copy.taskNo = taskNo;
@@ -500,8 +492,7 @@ public class CarbonLoadModel implements Serializable {
copyObj.csvHeader = header;
copyObj.csvHeaderColumns = csvHeaderColumns;
copyObj.csvDelimiter = delimiter;
- copyObj.complexDelimiterLevel1 = complexDelimiterLevel1;
- copyObj.complexDelimiterLevel2 = complexDelimiterLevel2;
+ copyObj.complexDelimiters = complexDelimiters;
copyObj.blocksID = blocksID;
copyObj.taskNo = taskNo;
copyObj.factTimeStamp = factTimeStamp;
@@ -631,7 +622,16 @@ public class CarbonLoadModel implements Serializable {
}
public String[] getDelimiters() {
- return new String[] { complexDelimiterLevel1, complexDelimiterLevel2 };
+ checkAndInitializeComplexDelimiterList();
+ String[] delimiters = new String[complexDelimiters.size()];
+ delimiters = complexDelimiters.toArray(delimiters);
+ return delimiters;
+ }
+
+ private void checkAndInitializeComplexDelimiterList() {
+ if (null == complexDelimiters) {
+ complexDelimiters = new ArrayList<>();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
index 4a29304..d02348d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -164,6 +164,8 @@ public class CarbonLoadModelBuilder {
String delimeter = optionsFinal.get("delimiter");
String complex_delimeter_level1 = optionsFinal.get("complex_delimiter_level_1");
String complex_delimeter_level2 = optionsFinal.get("complex_delimiter_level_2");
+ String complex_delimeter_level3 = optionsFinal.get("complex_delimiter_level_3");
+ String complex_delimeter_level4 = optionsFinal.get("complex_delimiter_level_4");
String all_dictionary_path = optionsFinal.get("all_dictionary_path");
String column_dict = optionsFinal.get("columndict");
validateDateTimeFormat(timestampformat, "TimestampFormat");
@@ -257,11 +259,14 @@ public class CarbonLoadModelBuilder {
if (delimeter.equalsIgnoreCase(complex_delimeter_level1) ||
complex_delimeter_level1.equalsIgnoreCase(complex_delimeter_level2) ||
- delimeter.equalsIgnoreCase(complex_delimeter_level2)) {
+ delimeter.equalsIgnoreCase(complex_delimeter_level2) ||
+ delimeter.equalsIgnoreCase(complex_delimeter_level3)) {
throw new InvalidLoadOptionException("Field Delimiter and Complex types delimiter are same");
} else {
- carbonLoadModel.setComplexDelimiterLevel1(complex_delimeter_level1);
- carbonLoadModel.setComplexDelimiterLevel2(complex_delimeter_level2);
+ carbonLoadModel.setComplexDelimiter(complex_delimeter_level1);
+ carbonLoadModel.setComplexDelimiter(complex_delimeter_level2);
+ carbonLoadModel.setComplexDelimiter(complex_delimeter_level3);
+ carbonLoadModel.setComplexDelimiter(complex_delimeter_level4);
}
// set local dictionary path, and dictionary file extension
carbonLoadModel.setAllDictPath(all_dictionary_path);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
index 2c5fa8b..759cf04 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.processing.loading.ComplexDelimitersEnum;
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
import org.apache.carbondata.processing.util.CarbonLoaderUtil;
@@ -111,13 +112,17 @@ public class LoadOption {
"all_dictionary_path",
Maps.getOrDefault(options, "all_dictionary_path", ""));
- optionsFinal.put(
- "complex_delimiter_level_1",
- Maps.getOrDefault(options,"complex_delimiter_level_1", "\\\001"));
+ optionsFinal.put("complex_delimiter_level_1",
+ Maps.getOrDefault(options, "complex_delimiter_level_1",
+ ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_1.value()));
- optionsFinal.put(
- "complex_delimiter_level_2",
- Maps.getOrDefault(options, "complex_delimiter_level_2", "\\\002"));
+ optionsFinal.put("complex_delimiter_level_2",
+ Maps.getOrDefault(options, "complex_delimiter_level_2",
+ ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_2.value()));
+
+ optionsFinal.put("complex_delimiter_level_3",
+ Maps.getOrDefault(options, "complex_delimiter_level_3",
+ ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_3.value()));
optionsFinal.put(
"dateformat",
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java
index 3964869..6ffea4f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java
@@ -16,6 +16,7 @@
*/
package org.apache.carbondata.processing.loading.parser;
+import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.metadata.datatype.DataType;
@@ -23,6 +24,7 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.processing.loading.parser.impl.ArrayParserImpl;
+import org.apache.carbondata.processing.loading.parser.impl.MapParserImpl;
import org.apache.carbondata.processing.loading.parser.impl.PrimitiveParserImpl;
import org.apache.carbondata.processing.loading.parser.impl.StructParserImpl;
@@ -35,7 +37,8 @@ public final class CarbonParserFactory {
* @param complexDelimiters
* @return
*/
- public static GenericParser createParser(CarbonColumn carbonColumn, String[] complexDelimiters,
+ public static GenericParser createParser(CarbonColumn carbonColumn,
+ ArrayList<String> complexDelimiters,
String nullFormat) {
return createParser(carbonColumn, complexDelimiters, nullFormat, 0);
}
@@ -51,23 +54,33 @@ public final class CarbonParserFactory {
* delimiters
* @return GenericParser
*/
- private static GenericParser createParser(CarbonColumn carbonColumn, String[] complexDelimiters,
- String nullFormat, int depth) {
+ private static GenericParser createParser(CarbonColumn carbonColumn,
+ ArrayList<String> complexDelimiters, String nullFormat, int depth) {
DataType dataType = carbonColumn.getDataType();
- if (DataTypes.isArrayType(dataType) || DataTypes.isMapType(dataType)) {
+ if (DataTypes.isArrayType(dataType)) {
List<CarbonDimension> listOfChildDimensions =
((CarbonDimension) carbonColumn).getListOfChildDimensions();
// Create array parser with complex delimiter
- ArrayParserImpl arrayParser = new ArrayParserImpl(complexDelimiters[depth], nullFormat);
+ ArrayParserImpl arrayParser = new ArrayParserImpl(complexDelimiters.get(depth), nullFormat);
for (CarbonDimension dimension : listOfChildDimensions) {
arrayParser.addChildren(createParser(dimension, complexDelimiters, nullFormat, depth + 1));
}
return arrayParser;
+ } else if (DataTypes.isMapType(dataType)) {
+ List<CarbonDimension> listOfChildDimensions =
+ ((CarbonDimension) carbonColumn).getListOfChildDimensions();
+ // Create map parser with complex delimiter and key-value delimiter
+ MapParserImpl mapParser = new MapParserImpl(complexDelimiters.get(depth), nullFormat,
+ complexDelimiters.get(depth + 1));
+ for (CarbonDimension dimension : listOfChildDimensions) {
+ mapParser.addChildren(createParser(dimension, complexDelimiters, nullFormat, depth + 1));
+ }
+ return mapParser;
} else if (DataTypes.isStructType(dataType)) {
List<CarbonDimension> dimensions =
((CarbonDimension) carbonColumn).getListOfChildDimensions();
// Create struct parser with complex delimiter
- StructParserImpl parser = new StructParserImpl(complexDelimiters[depth], nullFormat);
+ StructParserImpl parser = new StructParserImpl(complexDelimiters.get(depth), nullFormat);
for (CarbonDimension dimension : dimensions) {
parser.addChildren(createParser(dimension, complexDelimiters, nullFormat, depth + 1));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java
index c56691a..c27f0fa 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/ArrayParserImpl.java
@@ -32,11 +32,11 @@ import org.apache.commons.lang.ArrayUtils;
*/
public class ArrayParserImpl implements ComplexParser<ArrayObject> {
- private Pattern pattern;
+ protected Pattern pattern;
- private GenericParser child;
+ protected GenericParser child;
- private String nullFormat;
+ protected String nullFormat;
public ArrayParserImpl(String delimiter, String nullFormat) {
pattern = Pattern.compile(CarbonUtil.delimiterConverter(delimiter));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/MapParserImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/MapParserImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/MapParserImpl.java
new file mode 100644
index 0000000..e6814f8
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/MapParserImpl.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.loading.parser.impl;
+
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.carbondata.processing.loading.complexobjects.ArrayObject;
+
+import org.apache.commons.lang.ArrayUtils;
+
+
+public class MapParserImpl extends ArrayParserImpl {
+
+ private String keyValueDelimiter;
+
+ public MapParserImpl(String delimiter, String nullFormat, String keyValueDelimiter) {
+ super(delimiter, nullFormat);
+ this.keyValueDelimiter = keyValueDelimiter;
+ }
+
+ //The Key for Map will always be a PRIMITIVE type so Set<Object> here will work fine
+ //Only the first occurance of key will be added and the remaining will be skipped/ignored
+ @Override public ArrayObject parse(Object data) {
+ if (data != null) {
+ String value = data.toString();
+ if (!value.isEmpty() && !value.equals(nullFormat)) {
+ String[] split = pattern.split(value, -1);
+ if (ArrayUtils.isNotEmpty(split)) {
+ ArrayList<Object> array = new ArrayList<>();
+ Set<Object> set = new HashSet<>();
+ for (int i = 0; i < split.length; i++) {
+ Object currKey = split[i].split(keyValueDelimiter)[0];
+ if (set.add(currKey)) {
+ array.add(child.parse(split[i]));
+ }
+ }
+ return new ArrayObject(array.toArray());
+ }
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/RowParserImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/RowParserImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/RowParserImpl.java
index 00d8420..d0fe30b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/RowParserImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/RowParserImpl.java
@@ -16,6 +16,8 @@
*/
package org.apache.carbondata.processing.loading.parser.impl;
+import java.util.ArrayList;
+
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants;
@@ -34,8 +36,12 @@ public class RowParserImpl implements RowParser {
private int numberOfColumns;
public RowParserImpl(DataField[] output, CarbonDataLoadConfiguration configuration) {
- String[] complexDelimiters =
+ String[] tempComplexDelimiters =
(String[]) configuration.getDataLoadProperty(DataLoadProcessorConstants.COMPLEX_DELIMITERS);
+ ArrayList<String> complexDelimiters = new ArrayList<>();
+ for (int i = 0; i < tempComplexDelimiters.length; i++) {
+ complexDelimiters.add(tempComplexDelimiters[i]);
+ }
String nullFormat =
configuration.getDataLoadProperty(DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT)
.toString();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index c9adcdf..1241504 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -184,6 +184,7 @@ public class CarbonWriterBuilder {
!option.equalsIgnoreCase("timestampformat") &&
!option.equalsIgnoreCase("complex_delimiter_level_1") &&
!option.equalsIgnoreCase("complex_delimiter_level_2") &&
+ !option.equalsIgnoreCase("complex_delimiter_level_3") &&
!option.equalsIgnoreCase("quotechar") &&
!option.equalsIgnoreCase("escapechar")) {
throw new IllegalArgumentException("Unsupported option:" + option
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9b136c2/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
index 5a888ef..4dcb3ce 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
@@ -41,6 +41,7 @@ class RowStreamParserImp extends CarbonStreamParser {
var dateFormat: SimpleDateFormat = null
var complexDelimiterLevel1: String = null
var complexDelimiterLevel2: String = null
+ var complexDelimiterLevel3: String = null
var serializationNullFormat: String = null
override def initialize(configuration: Configuration, structType: StructType): Unit = {
@@ -54,6 +55,7 @@ class RowStreamParserImp extends CarbonStreamParser {
this.configuration.get(CarbonCommonConstants.CARBON_DATE_FORMAT))
this.complexDelimiterLevel1 = this.configuration.get("carbon_complex_delimiter_level_1")
this.complexDelimiterLevel2 = this.configuration.get("carbon_complex_delimiter_level_2")
+ this.complexDelimiterLevel3 = this.configuration.get("carbon_complex_delimiter_level_3")
this.serializationNullFormat =
this.configuration.get(DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT)
}
[08/21] carbondata git commit: [CARBONDATA-3005]Support Gzip as
column compressor
Posted by ra...@apache.org.
[CARBONDATA-3005]Support Gzip as column compressor
This PR is to add a new compressor "Gzip" and enhance the compressing capabilities offered by CarbonData.
User can now use gzip as the compressor for loading the data.
Gzip can be set at System Properties level or also for particular table.
This closes #2847
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8f57294d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8f57294d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8f57294d
Branch: refs/heads/branch-1.5
Commit: 8f57294d14532457d6fd3877a182f66055d338c2
Parents: 22bec10
Author: shardul-cr7 <sh...@gmail.com>
Authored: Tue Oct 23 17:27:47 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:33 2018 +0530
----------------------------------------------------------------------
.../compression/AbstractCompressor.java | 1 +
.../compression/CompressorFactory.java | 3 +-
.../datastore/compression/GzipCompressor.java | 134 +++++++++++++++++++
.../datastore/compression/ZstdCompressor.java | 5 -
.../dataload/TestLoadDataWithCompression.scala | 94 ++++++++++---
.../TestLoadWithSortTempCompressed.scala | 20 +++
6 files changed, 236 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8f57294d/core/src/main/java/org/apache/carbondata/core/datastore/compression/AbstractCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/AbstractCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/AbstractCompressor.java
index 0724bdc..c554dc6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/AbstractCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/AbstractCompressor.java
@@ -123,4 +123,5 @@ public abstract class AbstractCompressor implements Compressor {
return false;
}
+ @Override public boolean supportUnsafe() { return false; }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8f57294d/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
index f7d4e06..b7779ba 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
@@ -36,7 +36,8 @@ public class CompressorFactory {
public enum NativeSupportedCompressor {
SNAPPY("snappy", SnappyCompressor.class),
- ZSTD("zstd", ZstdCompressor.class);
+ ZSTD("zstd", ZstdCompressor.class),
+ GZIP("gzip", GzipCompressor.class);
private String name;
private Class<Compressor> compressorClass;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8f57294d/core/src/main/java/org/apache/carbondata/core/datastore/compression/GzipCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/GzipCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/GzipCompressor.java
new file mode 100644
index 0000000..b386913
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/GzipCompressor.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+
+/**
+ * Codec Class for performing Gzip Compression
+ */
+public class GzipCompressor extends AbstractCompressor {
+
+ @Override public String getName() {
+ return "gzip";
+ }
+
+ /**
+ * This method takes the Byte Array data and Compresses in gzip format
+ *
+ * @param data Data Byte Array passed for compression
+ * @return Compressed Byte Array
+ */
+ private byte[] compressData(byte[] data) {
+ int initialSize = (data.length / 2) == 0 ? data.length : data.length / 2;
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(initialSize);
+ try {
+ GzipCompressorOutputStream gzipCompressorOutputStream =
+ new GzipCompressorOutputStream(byteArrayOutputStream);
+ try {
+ /**
+ * Below api will write bytes from specified byte array to the gzipCompressorOutputStream
+ * The output stream will compress the given byte array.
+ */
+ gzipCompressorOutputStream.write(data);
+ } catch (IOException e) {
+ throw new RuntimeException("Error during Compression writing step ", e);
+ } finally {
+ gzipCompressorOutputStream.close();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Error during Compression step ", e);
+ }
+ return byteArrayOutputStream.toByteArray();
+ }
+
+ /**
+ * This method takes the Byte Array data and Decompresses in gzip format
+ *
+ * @param data Data Byte Array for Compression
+ * @param offset Start value of Data Byte Array
+ * @param length Size of Byte Array
+ * @return
+ */
+ private byte[] decompressData(byte[] data, int offset, int length) {
+ ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(data, offset, length);
+ ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
+ try {
+ GzipCompressorInputStream gzipCompressorInputStream =
+ new GzipCompressorInputStream(byteArrayInputStream);
+ int initialSize = (data.length * 2) < Integer.MAX_VALUE ? (data.length * 2) : data.length;
+ byte[] buffer = new byte[initialSize];
+ int len;
+ /**
+ * Reads the next byte of the data from the input stream and stores them into buffer
+ * Data is then read from the buffer and put into byteOutputStream from a offset.
+ */
+ while ((len = gzipCompressorInputStream.read(buffer)) != -1) {
+ byteOutputStream.write(buffer, 0, len);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Error during Decompression step ", e);
+ }
+ return byteOutputStream.toByteArray();
+ }
+
+ @Override public byte[] compressByte(byte[] unCompInput) {
+ return compressData(unCompInput);
+ }
+
+ @Override public byte[] compressByte(byte[] unCompInput, int byteSize) {
+ return compressData(unCompInput);
+ }
+
+ @Override public byte[] unCompressByte(byte[] compInput) {
+ return decompressData(compInput, 0, compInput.length);
+ }
+
+ @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
+ return decompressData(compInput, offset, length);
+ }
+
+ @Override public long rawUncompress(byte[] input, byte[] output) {
+ //gzip api doesnt have rawUncompress yet.
+ throw new RuntimeException("Not implemented rawUcompress for gzip yet");
+ }
+
+ @Override public long maxCompressedLength(long inputSize) {
+ // Check if input size is lower than the max possible size
+ if (inputSize < Integer.MAX_VALUE) {
+ return inputSize;
+ } else {
+ throw new RuntimeException("compress input oversize for gzip");
+ }
+ }
+
+ @Override public int unCompressedLength(byte[] data, int offset, int length) {
+ //gzip api doesnt have UncompressedLength
+ throw new RuntimeException("Unsupported operation Exception");
+ }
+
+ @Override public int rawUncompress(byte[] data, int offset, int length, byte[] output) {
+ //gzip api doesnt have rawUncompress yet.
+ throw new RuntimeException("Not implemented rawUcompress for gzip yet");
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8f57294d/core/src/main/java/org/apache/carbondata/core/datastore/compression/ZstdCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ZstdCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ZstdCompressor.java
index 8523c46..56faa20 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ZstdCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ZstdCompressor.java
@@ -70,11 +70,6 @@ public class ZstdCompressor extends AbstractCompressor {
* currently java version of zstd does not support this feature.
* It may support it in upcoming release 1.3.5-3, then we can optimize this accordingly.
*/
- @Override
- public boolean supportUnsafe() {
- return false;
- }
-
@Override public int unCompressedLength(byte[] data, int offset, int length) {
throw new RuntimeException("Unsupported operation Exception");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8f57294d/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala
index 6ec44e7..b12e269 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala
@@ -168,6 +168,7 @@ class TestLoadDataWithCompression extends QueryTest with BeforeAndAfterEach with
private val tableName = "load_test_with_compressor"
private var executorService: ExecutorService = _
private val csvDataDir = s"$integrationPath/spark2/target/csv_load_compression"
+ private val compressors = Array("snappy","zstd","gzip")
override protected def beforeAll(): Unit = {
executorService = Executors.newFixedThreadPool(3)
@@ -252,50 +253,94 @@ class TestLoadDataWithCompression extends QueryTest with BeforeAndAfterEach with
""".stripMargin)
}
- test("test data loading with snappy compressor and offheap") {
+ test("test data loading with different compressors and offheap") {
+ for(comp <- compressors){
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, comp)
+ createTable()
+ loadData()
+ checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(8)))
+ }
+ }
+
+ test("test data loading with different compressors and onheap") {
+ for(comp <- compressors){
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, comp)
+ createTable()
+ loadData()
+ checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(8)))
+ }
+ }
+
+ test("test current zstd compressor on legacy store with snappy") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "snappy")
createTable()
loadData()
- checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(8)))
- }
- test("test data loading with zstd compressor and offheap") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "zstd")
+ loadData()
+ checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(16)))
+ }
+
+ test("test current gzip compressor on legacy store with snappy") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "snappy")
createTable()
loadData()
- checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(8)))
+
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "gzip")
+ loadData()
+ checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(16)))
}
- test("test data loading with zstd compressor and onheap") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false")
+ test("test current snappy compressor on legacy store with zstd") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "zstd")
createTable()
loadData()
- checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(8)))
- }
- test("test current zstd compressor on legacy store with snappy") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "snappy")
+ loadData()
+ checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(16)))
+ }
+
+ test("test current snappy compressor on legacy store with gzip") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "gzip")
createTable()
loadData()
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "ZSTD")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "snappy")
loadData()
checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(16)))
}
- test("test current snappy compressor on legacy store with zstd") {
+ test("test current gzip compressor on legacy store with zstd") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "zstd")
createTable()
loadData()
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "snappy")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "gzip")
+ loadData()
+ checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(16)))
+ }
+
+ test("test current zstd compressor on legacy store with gzip") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "gzip")
+ createTable()
+ loadData()
+
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "zstd")
loadData()
checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(16)))
}
@@ -311,7 +356,7 @@ class TestLoadDataWithCompression extends QueryTest with BeforeAndAfterEach with
loadData()
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "zstd")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "gzip")
loadData()
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
@@ -416,11 +461,17 @@ class TestLoadDataWithCompression extends QueryTest with BeforeAndAfterEach with
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, if (Random.nextBoolean()) "snappy" else "zstd")
}
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "gzip")
+ future = compactAsync()
+ while (!future.isDone) {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, if (Random.nextBoolean()) "snappy" else "gzip")
+ }
+
checkAnswer(sql(s"SELECT COUNT(*) FROM $tableName"), Seq(Row(lineNum * 2)))
checkAnswer(sql(s"SELECT stringDictField, stringSortField FROM $tableName WHERE stringDictField='stringDict1'"), Seq(Row("stringDict1", "stringSort1"), Row("stringDict1", "stringSort1")))
}
- test("test creating table with specified compressor") {
+ test("test creating table with specified zstd compressor") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
// the system configuration for compressor is snappy
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "snappy")
@@ -433,6 +484,19 @@ class TestLoadDataWithCompression extends QueryTest with BeforeAndAfterEach with
assertResult("zstd")(tableColumnCompressor)
}
+ test("test creating table with specified gzip compressor") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ // the system configuration for compressor is snappy
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.COMPRESSOR, "snappy")
+ // create table with gzip as compressor
+ createTable(columnCompressor = "gzip")
+ loadData()
+ checkAnswer(sql(s"SELECT count(*) FROM $tableName"), Seq(Row(8)))
+ val carbonTable = CarbonEnv.getCarbonTable(Option("default"), tableName)(sqlContext.sparkSession)
+ val tableColumnCompressor = carbonTable.getTableInfo.getFactTable.getTableProperties.get(CarbonCommonConstants.COMPRESSOR)
+ assert("gzip".equalsIgnoreCase(tableColumnCompressor))
+ }
+
test("test creating table with unsupported compressor") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
// the system configuration for compressor is snappy
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8f57294d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala
index 21affee..5d9f05c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadWithSortTempCompressed.scala
@@ -137,6 +137,26 @@ class TestLoadWithSortTempCompressed extends QueryTest
originOffHeapStatus)
}
+ test("test data load for simple table with sort temp compressed with gzip" +
+ " and off-heap sort enabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "gzip")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "true")
+ testSimpleTable()
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ originOffHeapStatus)
+ }
+
+ test("test data load for simple table with sort temp compressed with gzip" +
+ " and off-heap sort disabled") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR,
+ "gzip")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false")
+ testSimpleTable()
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+ originOffHeapStatus)
+ }
+
private def testComplexTable(): Unit = {
// note: following tests are copied from `TestComplexTypeQuery`
sql(
[17/21] carbondata git commit: [CARBONDATA-3153] Complex delimiters
change
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala
index f5596f2..e3c2d88 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala
@@ -2323,23 +2323,23 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
stringBuilder.append(index.toString + "abc,name_" + index
+ ",city_" + index + "," + (10000.00 * index).toString + ",0.01,80.01" +
",1990-01-01,2010-01-01 10:01:01,2010-01-01 10:01:01" +
- ",school_" + index + ":school_" + index + index + "$" + index)
+ ",school_" + index + "\002school_" + index + index + "\001" + index)
} else if (index == 9) {
stringBuilder.append(index.toString + ",name_" + index
+ ",city_" + index + "," + (10000.00 * index).toString + ",0.04,80.04" +
",1990-01-04,2010-01-04 10:01:01,2010-01-04 10:01:01" +
- ",school_" + index + ":school_" + index + index + "$" + index)
+ ",school_" + index + "\002school_" + index + index + "\001" + index)
} else {
stringBuilder.append(index.toString + ",name_" + index
+ ",city_" + index + "," + (10000.00 * index).toString + ",0.01,80.01" +
",1990-01-01,2010-01-01 10:01:01,2010-01-01 10:01:01" +
- ",school_" + index + ":school_" + index + index + "$" + index)
+ ",school_" + index + "\002school_" + index + index + "\001" + index)
}
} else {
stringBuilder.append(index.toString + ",name_" + index
+ ",city_" + index + "," + (10000.00 * index).toString + ",0.01,80.01" +
",1990-01-01,2010-01-01 10:01:01,2010-01-01 10:01:01" +
- ",school_" + index + ":school_" + index + index + "$" + index)
+ ",school_" + index + "\002school_" + index + index + "\001" + index)
}
stringBuilder.append("\n")
}
@@ -2474,7 +2474,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
"1990-01-01",
"2010-01-01 10:01:01",
"2010-01-01 10:01:01",
- "school_" + id + ":school_" + id + id + "$" + id)
+ "school_" + id + "\002school_" + id + id + "\001" + id)
}
spark.createDataFrame(csvRDD).toDF(
"id", "name", "city", "salary", "tax", "percent", "birthday", "register", "updated", "file")
@@ -2489,7 +2489,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
"1990-01-01",
"2010-01-01 10:01:01",
"2010-01-01 10:01:01",
- "school_" + id + ":school_" + id + id + "$" + id)
+ "school_" + id + "\002school_" + id + id + "\001" + id)
}
spark.createDataFrame(csvRDD).toDF(
"id", "salary", "tax", "percent", "birthday", "register", "updated", "file")
@@ -2594,11 +2594,8 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
def executeBatchLoad(tableName: String): Unit = {
sql(
- s"""
- | LOAD DATA LOCAL INPATH '$dataFilePath'
- | INTO TABLE streaming.$tableName
- | OPTIONS('HEADER'='true')
- """.stripMargin)
+ s"LOAD DATA LOCAL INPATH '$dataFilePath' INTO TABLE streaming.$tableName OPTIONS" +
+ "('HEADER'='true','COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
}
def wrap(array: Array[String]) = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
index 9beee59..985b9d9 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.spark.carbondata
+package org.apache.spark.carbondatafalse
import java.io.{File, PrintWriter}
import java.math.BigDecimal
@@ -29,7 +29,7 @@ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, Ignore}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.statusmanager.{FileFormat, SegmentStatus}
@@ -42,6 +42,7 @@ case class StreamData(id: Integer, name: String, city: String, salary: java.lang
register: String, updated: String,
file: FileElement)
+@Ignore
class TestStreamingTableWithRowParser extends QueryTest with BeforeAndAfterAll {
private val spark = sqlContext.sparkSession
@@ -419,7 +420,7 @@ class TestStreamingTableWithRowParser extends QueryTest with BeforeAndAfterAll {
continueSeconds = 20,
generateBadRecords = true,
badRecordAction = "force",
- autoHandoff = false
+ autoHandoff = true
)
// non-filter
@@ -434,7 +435,7 @@ class TestStreamingTableWithRowParser extends QueryTest with BeforeAndAfterAll {
assert(result(50).getInt(0) == 100000001)
assert(result(50).getString(1) == "batch_1")
assert(result(50).getStruct(9).getInt(1) == 20)
-
+ sql("select * from streaming1.stream_table_filter_complex where id = 1").show
// filter
checkAnswer(
sql("select * from stream_table_filter_complex where id = 1"),
@@ -772,7 +773,8 @@ class TestStreamingTableWithRowParser extends QueryTest with BeforeAndAfterAll {
fields(6), fields(7), fields(8), file)
}
}
- } }
+ }
+ }
// Write data from socket stream to carbondata file
qry = readSocketDF.writeStream
@@ -903,11 +905,8 @@ class TestStreamingTableWithRowParser extends QueryTest with BeforeAndAfterAll {
def executeBatchLoad(tableName: String): Unit = {
sql(
- s"""
- | LOAD DATA LOCAL INPATH '$dataFilePath'
- | INTO TABLE streaming1.$tableName
- | OPTIONS('HEADER'='true')
- """.stripMargin)
+ s"LOAD DATA LOCAL INPATH '$dataFilePath' INTO TABLE streaming1.$tableName OPTIONS" +
+ "('HEADER'='true','COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
}
def wrap(array: Array[String]) = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
index b53976a..2c5fa8b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
@@ -113,11 +113,11 @@ public class LoadOption {
optionsFinal.put(
"complex_delimiter_level_1",
- Maps.getOrDefault(options,"complex_delimiter_level_1", "$"));
+ Maps.getOrDefault(options,"complex_delimiter_level_1", "\\\001"));
optionsFinal.put(
"complex_delimiter_level_2",
- Maps.getOrDefault(options, "complex_delimiter_level_2", ":"));
+ Maps.getOrDefault(options, "complex_delimiter_level_2", "\\\002"));
optionsFinal.put(
"dateformat",
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index 2257639..c9adcdf 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -166,8 +166,8 @@ public class CarbonWriterBuilder {
* c. bad_record_path -- ""
* d. dateformat -- "" , uses from carbon.properties file
* e. timestampformat -- "", uses from carbon.properties file
- * f. complex_delimiter_level_1 -- "$"
- * g. complex_delimiter_level_2 -- ":"
+ * f. complex_delimiter_level_1 -- "\001"
+ * g. complex_delimiter_level_2 -- "\002"
* h. quotechar -- "\""
* i. escapechar -- "\\"
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
index d957ff6..58b9b59 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
@@ -492,7 +492,7 @@ public class CSVCarbonWriterTest {
CarbonWriterBuilder builder = CarbonWriter.builder().taskNo(5).outputPath(path);
CarbonWriter writer = builder.withCsvInput(new Schema(new Field[] {structType})).writtenBy("CSVCarbonWriterTest").build();
for (int i = 0; i < 15; i++) {
- String[] row = new String[] { "robot" + (i % 10)+"$" + i+ "$" + i + "." + i };
+ String[] row = new String[] { "robot" + (i % 10)+"\001" + i+ "\001" + i + "." + i };
writer.write(row);
}
writer.close();
@@ -531,7 +531,7 @@ public class CSVCarbonWriterTest {
CarbonWriterBuilder builder = CarbonWriter.builder().taskNo(5).outputPath(path);
CarbonWriter writer = builder.withCsvInput(new Schema(new Field[] {structType1, structType2})).writtenBy("CSVCarbonWriterTest").build();
for (int i = 0; i < 15; i++) {
- String[] row = new String[] { "1.0$2.0$3.0", "1$2$3" };
+ String[] row = new String[] { "1.0\0012.0\0013.0", "1\0012\0013" };
writer.write(row);
}
writer.close();
[06/21] carbondata git commit: [CARBONDATA-2840] Added SDV testcases
for Complex DataType Support
Posted by ra...@apache.org.
[CARBONDATA-2840] Added SDV testcases for Complex DataType Support
Added SDV testcases for Complex DataType Support
This closes #2621
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/22bec100
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/22bec100
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/22bec100
Branch: refs/heads/branch-1.5
Commit: 22bec1001dd348e543d953cdb2eefe825700b877
Parents: ee8bfd0
Author: Indhumathi27 <in...@gmail.com>
Authored: Wed Aug 8 15:33:36 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:33 2018 +0530
----------------------------------------------------------------------
.../sdv/generated/ComplexDataTypeTestCase.scala | 395 +++++++++++++++++++
.../cluster/sdv/suite/SDVSuites.scala | 1 +
2 files changed, 396 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/22bec100/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala
new file mode 100644
index 0000000..723c4f7
--- /dev/null
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala
@@ -0,0 +1,395 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, File, InputStream}
+import java.sql.Timestamp
+
+import scala.collection.mutable
+
+import org.apache.avro.file.DataFileWriter
+import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord}
+import org.apache.avro.io.{DecoderFactory, Encoder}
+import org.apache.commons.io.FileUtils
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.test.TestQueryExecutor
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.sdk.file.CarbonWriter
+
+/**
+ * Test Class for ComplexDataTypeTestCase to verify all scenerios
+ */
+
+class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val filePath = TestQueryExecutor.integrationPath + "/spark-common-test/src/test/resources"
+ val writerPath =
+ s"${ resourcesPath }" + "/SparkCarbonFileFormat/WriterOutputComplex/"
+
+ override def beforeAll(): Unit = {
+ FileUtils.deleteDirectory(new File(writerPath))
+ sql("DROP TABLE IF EXISTS complexcarbontable")
+ sql("DROP TABLE IF EXISTS test")
+ sql("DROP TABLE IF EXISTS sdkOutputTable")
+ }
+
+ override def afterAll(): Unit = {
+ FileUtils.deleteDirectory(new File(writerPath))
+ sql("DROP TABLE IF EXISTS complexcarbontable")
+ sql("DROP TABLE IF EXISTS test")
+ sql("DROP TABLE IF EXISTS sdkOutputTable")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ }
+
+ // check create table with complex data type and insert into complex table
+ test("test Complex_DataType-001") {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ sql("DROP TABLE IF EXISTS test")
+ sql(
+ "create table test(person struct<detail:struct<id:int,name:string,height:double," +
+ "status:boolean,dob:date,dobt:timestamp>>) stored by 'carbondata'")
+ sql("insert into test values('1:abc:4.30:true:2017/08/09:2017/08/09')")
+ checkAnswer(sql("select * from test"),
+ Seq(Row(Row(Row(1, "abc", 4.3, true, java.sql.Date.valueOf("2017-08-09"),
+ Timestamp.valueOf("2017-08-09 00:00:00.0"))))))
+ sql("DROP TABLE IF EXISTS test")
+ sql(
+ "create table test(p1 array<int>,p2 array<string>,p3 array<double>,p4 array<boolean>,p5 " +
+ "array<date>,p6 array<timestamp>) stored by 'carbondata'")
+ sql(
+ "insert into test values('1$2$3','abc$def$mno','4.30$4.60$5.20','true$true$false'," +
+ "'2017/08/09$2017/08/09$2017/07/07','2017/08/09$2017/08/09$2017/07/07')")
+ checkAnswer(sql("select * from test"),
+ Seq(Row(mutable.WrappedArray.make(Array(1, 2, 3)),
+ mutable.WrappedArray.make(Array("abc", "def", "mno")),
+ mutable.WrappedArray.make(Array(4.3, 4.6, 5.2)),
+ mutable.WrappedArray.make(Array(true, true, false)),
+ mutable.WrappedArray
+ .make(Array(java.sql.Date.valueOf("2017-08-09"),
+ java.sql.Date.valueOf("2017-08-09"),
+ java.sql.Date.valueOf("2017-07-07"))),
+ mutable.WrappedArray
+ .make(Array(Timestamp.valueOf("2017-08-09 00:00:00.0"),
+ Timestamp.valueOf("2017-08-09 00:00:00.0"),
+ Timestamp.valueOf("2017-07-07 00:00:00.0"))))))
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ }
+
+ // check create table with complex data type and load data into complex table
+ test("test Complex_DataType-002") {
+ sql("drop table if exists complexcarbontable")
+ sql("create table complexcarbontable(deviceInformationId int, channelsId string," +
+ "ROMSize string, purchasedate string, mobile struct<imei:string, imsi:string>," +
+ "MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
+ "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>," +
+ "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
+ "double,contractNumber double) " +
+ "STORED BY 'org.apache.carbondata.format'")
+ sql(
+ s"LOAD DATA local inpath '$filePath/complexdata.csv' INTO table " +
+ "complexcarbontable " +
+ "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId," +
+ "ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber'," +
+ "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
+ checkAnswer(sql("select * from complexcarbontable where deviceInformationId = 1"),
+ Seq(Row(1, "109", "4ROM size", "29-11-2015", Row("1AA1", "2BB1"),
+ mutable.WrappedArray.make(Array("MAC1", "MAC2", "MAC3")),
+ mutable.WrappedArray
+ .make(Array(Row(7, "Chinese", "Hubei Province", "yichang", "yichang", "yichang"),
+ Row(7, "India", "New Delhi", "delhi", "delhi", "delhi"))),
+ Row("29-11-2015", mutable.WrappedArray.make(Array("29-11-2015", "29-11-2015"))),
+ 109.0, 2738.562)))
+ }
+
+ // check create table with complex data type with dictionary_include columns and insert into
+ // into complex table
+ test("test Complex_DataType-003") {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ sql("DROP TABLE IF EXISTS test")
+ sql(
+ "create table test(person struct<detail:struct<id:int,name:string,height:double," +
+ "status:boolean,dob:date,dobt:timestamp>>) stored by 'carbondata' tblproperties" +
+ "('dictionary_include'='person')")
+ sql("insert into test values('1:abc:4.30:true:2017/08/09:2017/08/09')")
+ checkAnswer(sql("select * from test"),
+ Seq(Row(Row(Row(1,
+ "abc", 4.3, true, java.sql.Date.valueOf("2017-08-09"),
+ Timestamp.valueOf("2017-08-09 00:00:00.0"))))))
+ sql("DROP TABLE IF EXISTS test")
+ sql(
+ "create table test(p1 array<int>,p2 array<string>,p3 array<double>,p4 array<boolean>,p5 " +
+ "array<date>,p6 array<timestamp>) stored by 'carbondata' tblproperties" +
+ "('dictionary_include'='p1,p2,p3,p4,p5,p6')")
+ sql(
+ "insert into test values('1$2$3','abc$def$mno','4.30$4.60$5.20','true$true$false'," +
+ "'2017/08/09$2017/08/09$2017/07/07','2017/08/09$2017/08/09$2017/07/07')")
+ checkAnswer(sql("select * from test"),
+ Seq(Row(mutable.WrappedArray.make(Array(1, 2, 3)),
+ mutable.WrappedArray.make(Array("abc", "def", "mno")),
+ mutable.WrappedArray.make(Array(4.3, 4.6, 5.2)),
+ mutable.WrappedArray.make(Array(true, true, false)),
+ mutable.WrappedArray
+ .make(Array(java.sql.Date.valueOf("2017-08-09"),
+ java.sql.Date.valueOf("2017-08-09"),
+ java.sql.Date.valueOf("2017-07-07"))),
+ mutable.WrappedArray
+ .make(Array(Timestamp.valueOf("2017-08-09 00:00:00.0"),
+ Timestamp.valueOf("2017-08-09 00:00:00.0"),
+ Timestamp.valueOf("2017-07-07 00:00:00.0"))))))
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ }
+
+ // check ctas with complex datatype table
+ test("test Complex_DataType-004") {
+ sql("drop table if exists complexcarbontable")
+ sql("create table complexcarbontable(deviceInformationId int, channelsId string," +
+ "ROMSize string, purchasedate string, mobile struct<imei:string, imsi:string>," +
+ "MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
+ "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>," +
+ "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
+ "double,contractNumber double) " +
+ "STORED BY 'org.apache.carbondata.format'")
+ sql(
+ s"LOAD DATA local inpath '$filePath/complexdata.csv' INTO table " +
+ "complexcarbontable " +
+ "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId," +
+ "ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber'," +
+ "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
+ checkAnswer(sql("select count(*) from complexcarbontable"), Seq(Row(100)))
+ sql("DROP TABLE IF EXISTS test")
+ sql("create table test stored by 'carbondata' as select * from complexcarbontable")
+ checkAnswer(sql("select count(*) from test"), Seq(Row(100)))
+ }
+
+ //check projection pushdown with complex- STRUCT data type
+ test("test Complex_DataType-005") {
+ sql("DROP TABLE IF EXISTS complexcarbontable")
+ sql(
+ "create table complexcarbontable (roll int,a struct<b:int,c:string,d:int,e:string," +
+ "f:struct<g:int," +
+ "h:string,i:int>,j:int>) stored " +
+ "by " +
+ "'carbondata'")
+ sql("insert into complexcarbontable values(1,'1$abc$2$efg$3:mno:4$5')")
+ sql("insert into complexcarbontable values(2,'1$abc$2$efg$3:mno:4$5')")
+ sql("insert into complexcarbontable values(3,'1$abc$2$efg$3:mno:4$5')")
+ checkAnswer(sql("select a.b from complexcarbontable"), Seq(Row(1), Row(1), Row(1)))
+ checkAnswer(sql("select a.c from complexcarbontable"), Seq(Row("abc"), Row("abc"), Row("abc")))
+ checkAnswer(sql("select a.d from complexcarbontable"), Seq(Row(2), Row(2), Row(2)))
+ checkAnswer(sql("select a.e from complexcarbontable"), Seq(Row("efg"), Row("efg"), Row("efg")))
+ checkAnswer(sql("select a.f from complexcarbontable"),
+ Seq(Row(Row(3, "mno", 4)), Row(Row(3, "mno", 4)), Row(Row(3, "mno", 4))))
+ checkAnswer(sql("select a.f.g from complexcarbontable"), Seq(Row(3), Row(3), Row(3)))
+ checkAnswer(sql("select a.f.h from complexcarbontable"),
+ Seq(Row("mno"), Row("mno"), Row("mno")))
+ checkAnswer(sql("select a.f.i from complexcarbontable"), Seq(Row(4), Row(4), Row(4)))
+ checkAnswer(sql("select a.f.g,a.f.h,a.f.i from complexcarbontable"),
+ Seq(Row(3, "mno", 4), Row(3, "mno", 4), Row(3, "mno", 4)))
+ checkAnswer(sql("select a.b,a.f from complexcarbontable"),
+ Seq(Row(1, Row(3, "mno", 4)), Row(1, Row(3, "mno", 4)), Row(1, Row(3, "mno", 4))))
+ checkAnswer(sql("select a.c,a.f from complexcarbontable"),
+ Seq(Row("abc", Row(3, "mno", 4)), Row("abc", Row(3, "mno", 4)), Row("abc", Row(3, "mno", 4))))
+ checkAnswer(sql("select a.d,a.f from complexcarbontable"),
+ Seq(Row(2, Row(3, "mno", 4)), Row(2, Row(3, "mno", 4)), Row(2, Row(3, "mno", 4))))
+ checkAnswer(sql("select a.j from complexcarbontable"), Seq(Row(5), Row(5), Row(5)))
+ checkAnswer(sql("select * from complexcarbontable"),
+ Seq(Row(1, Row(1, "abc", 2, "efg", Row(3, "mno", 4), 5)),
+ Row(2, Row(1, "abc", 2, "efg", Row(3, "mno", 4), 5)),
+ Row(3, Row(1, "abc", 2, "efg", Row(3, "mno", 4), 5))))
+ checkAnswer(sql("select *,a from complexcarbontable"),
+ Seq(Row(1,
+ Row(1, "abc", 2, "efg", Row(3, "mno", 4), 5),
+ Row(1, "abc", 2, "efg", Row(3, "mno", 4), 5)),
+ Row(2,
+ Row(1, "abc", 2, "efg", Row(3, "mno", 4), 5),
+ Row(1, "abc", 2, "efg", Row(3, "mno", 4), 5)),
+ Row(3,
+ Row(1, "abc", 2, "efg", Row(3, "mno", 4), 5),
+ Row(1, "abc", 2, "efg", Row(3, "mno", 4), 5))))
+ }
+
+ // check create table with complex datatype columns and insert into table and apply filters
+ test("test Complex_DataType-006") {
+ sql("DROP TABLE IF EXISTS test")
+ sql("create table test(id int,a struct<b:int,c:int>) stored by 'carbondata'")
+ sql("insert into test values(1,'2$3')")
+ sql("insert into test values(3,'5$3')")
+ sql("insert into test values(2,'4$5')")
+ checkAnswer(sql("select a.b from test where id=3"), Seq(Row(5)))
+ checkAnswer(sql("select a.b from test where a.c!=3"), Seq(Row(4)))
+ checkAnswer(sql("select a.b from test where a.c=3"), Seq(Row(5), Row(2)))
+ checkAnswer(sql("select a.b from test where id=1 or !a.c=3"), Seq(Row(4), Row(2)))
+ checkAnswer(sql("select a.b from test where id=3 or a.c=3"), Seq(Row(5), Row(2)))
+ }
+
+ // check create table with complex datatype columns and perform insertoverwrite
+ test("test Complex_DataType-007") {
+ sql("drop table if exists complexcarbontable")
+ sql("create table complexcarbontable(deviceInformationId int, channelsId string," +
+ "ROMSize string, purchasedate string, mobile struct<imei:string, imsi:string>," +
+ "MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
+ "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>," +
+ "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
+ "double,contractNumber double) " +
+ "STORED BY 'org.apache.carbondata.format'")
+ sql(
+ s"LOAD DATA local inpath '$filePath/complexdata.csv' INTO table " +
+ "complexcarbontable " +
+ "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId," +
+ "ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber'," +
+ "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
+ checkAnswer(sql("select count(*) from complexcarbontable"), Seq(Row(100)))
+ sql("DROP TABLE IF EXISTS test")
+ sql("create table test(deviceInformationId int, channelsId string," +
+ "ROMSize string, purchasedate string, mobile struct<imei:string, imsi:string>," +
+ "MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
+ "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>," +
+ "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
+ "double,contractNumber double) " +
+ "STORED BY 'org.apache.carbondata.format'")
+ sql("insert overwrite table test select * from complexcarbontable")
+ checkAnswer(sql("select count(*) from test"), Seq(Row(100)))
+ }
+
+ // check create complex table and insert null values
+ test("test Complex_DataType-008") {
+ sql("drop table if exists complexcarbontable")
+ sql(
+ "create table complexcarbontable(roll int, student struct<id:int,name:string," +
+ "marks:array<int>>) " +
+ "stored by 'carbondata'")
+ sql("insert into complexcarbontable values(1,'null$abc$null:null:null')")
+ checkAnswer(sql("select * from complexcarbontable"),
+ Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
+ }
+
+ //check create table with complex double and insert bigger value and check
+ test("test Complex_DataType-009") {
+ sql("Drop table if exists complexcarbontable")
+ sql(
+ "create table complexcarbontable(array1 array<struct<double1:double,double2:double," +
+ "double3:double>>) " +
+ "stored by 'carbondata'")
+ sql(
+ "insert into complexcarbontable values" +
+ "('14.35:400000.35:1.7976931348623157$67890985.888:65.5656:200')," +
+ "('20.25:50000.25:4.945464565654656546546546324$10000000:300000:3000')")
+ checkExistence(sql("select * from complexcarbontable"), true, "1.0E7,300000.0,3000.0")
+ sql("Drop table if exists complexcarbontable")
+ sql(
+ "create table complexcarbontable(struct_arr struct<array_db1:array<double>>) stored by " +
+ "'carbondata'")
+ sql("insert into complexcarbontable values('5555555.9559:12345678991234567:3444.999')")
+ checkExistence(sql("select * from complexcarbontable"),
+ true,
+ "5555555.9559, 1.2345678991234568E16, 3444.999")
+ }
+
+ // check create table with complex data type through SDK
+ test("test Complex_DataType-010") {
+ val mySchema =
+ """ {
+ | "name": "address",
+ | "type": "record",
+ | "fields": [
+ | {
+ | "name": "name",
+ | "type": "string"
+ | },
+ | {
+ | "name": "age",
+ | "type": "int"
+ | },
+ | {
+ | "name" :"my_address",
+ | "type" :{
+ | "name": "my_address",
+ | "type": "record",
+ | "fields": [
+ | {
+ | "name": "Temperaturetest",
+ | "type": "double"
+ | }
+ | ]
+ | }
+ | }
+ | ]
+ |} """.stripMargin
+
+ val jsonvalue =
+ """{
+ |"name" :"abcde",
+ |"age" :34,
+ |"my_address" :{ "Temperaturetest" :100 }
+ |}
+ """.stripMargin
+ val pschema = org.apache.avro.Schema.parse(mySchema)
+ val records = jsonToAvro(jsonvalue, mySchema)
+ val writer = CarbonWriter.builder().outputPath(writerPath).buildWriterForAvroInput(pschema)
+ writer.write(records)
+ writer.close()
+ sql("DROP TABLE IF EXISTS sdkOutputTable")
+ sql(
+ s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY 'carbondata' LOCATION
+ |'$writerPath' """.stripMargin)
+
+ checkAnswer(sql("select * from sdkOutputTable"), Seq(Row("abcde", 34, Row(100.0))))
+ }
+
+ def jsonToAvro(json: String, avroSchema: String): GenericRecord = {
+ var input: InputStream = null
+ var writer: DataFileWriter[GenericRecord] = null
+ var output: ByteArrayOutputStream = null
+ try {
+ val schema = new org.apache.avro.Schema.Parser().parse(avroSchema)
+ val reader = new GenericDatumReader[GenericRecord](schema)
+ input = new ByteArrayInputStream(json.getBytes())
+ output = new ByteArrayOutputStream()
+ val din = new DataInputStream(input)
+ writer = new DataFileWriter[GenericRecord](new GenericDatumWriter[GenericRecord]())
+ writer.create(schema, output)
+ val decoder = DecoderFactory.get().jsonDecoder(schema, din)
+ var datum: GenericRecord = reader.read(null, decoder)
+ return datum
+ } finally {
+ input.close()
+ writer.close()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/22bec100/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
index 3993042..d117d69 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala
@@ -155,6 +155,7 @@ class SDVSuites3 extends Suites with BeforeAndAfterAll {
new TestPartitionWithGlobalSort ::
new SDKwriterTestCase ::
new SetParameterTestCase ::
+ new ComplexDataTypeTestCase ::
new PartitionWithPreAggregateTestCase ::
new CreateTableWithLocalDictionaryTestCase ::
new LoadTableWithLocalDictionaryTestCase :: Nil
[21/21] carbondata git commit: [CARBONDATA-3102] Fix
NoClassDefFoundError when use thriftServer and beeline to read/write data
from/to S3
Posted by ra...@apache.org.
[CARBONDATA-3102] Fix NoClassDefFoundError when use thriftServer and beeline to read/write data from/to S3
This PR fix NoClassDefFoundError when use thriftServer and beeline to use table on cloud storage
This closes #2925
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ef787200
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ef787200
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ef787200
Branch: refs/heads/branch-1.5
Commit: ef7872009a09b7b2fb37991a9765d4f865d98078
Parents: 313ba0e
Author: xubo245 <xu...@huawei.com>
Authored: Thu Nov 15 20:36:12 2018 +0800
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:34 2018 +0530
----------------------------------------------------------------------
.../apache/carbondata/benchmark/SimpleQueryBenchmark.scala | 2 +-
.../scala/org/apache/carbondata/examples/S3Example.scala | 8 ++++----
integration/hive/pom.xml | 4 ++--
integration/presto/pom.xml | 2 +-
integration/spark2/pom.xml | 5 +++++
pom.xml | 1 +
6 files changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef787200/examples/spark2/src/main/scala/org/apache/carbondata/benchmark/SimpleQueryBenchmark.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/benchmark/SimpleQueryBenchmark.scala b/examples/spark2/src/main/scala/org/apache/carbondata/benchmark/SimpleQueryBenchmark.scala
index ce69c66..595af20 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/benchmark/SimpleQueryBenchmark.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/benchmark/SimpleQueryBenchmark.scala
@@ -273,7 +273,7 @@ object SimpleQueryBenchmark {
}
}
- // run testcases and print comparison result
+ // run test cases and print comparison result
private def runTest(spark: SparkSession, table1: String, table2: String): Unit = {
val formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
val date = new Date
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef787200/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
index d3d0a37..9cc43d0 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
@@ -157,8 +157,8 @@ object S3Example {
}
def getSparkMaster(args: Array[String]): String = {
- if (args.length == 5) args(4)
- else if (args(3).contains("spark:") || args(3).contains("mesos:")) args(3)
- else "local"
- }
+ if (args.length == 5) args(4)
+ else if (args(3).contains("spark:") || args(3).contains("mesos:")) args(3)
+ else "local"
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef787200/integration/hive/pom.xml
----------------------------------------------------------------------
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index 0128241..7b8b06e 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -86,12 +86,12 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.3.4</version>
+ <version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.3-alpha1</version>
+ <version>${httpcore.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef787200/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 1c1b91c..828696b 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -543,7 +543,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.2</version>
+ <version>${httpcore.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef787200/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index f127dce..aea3da3 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -129,6 +129,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ </dependency>
+ <dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
<version>0.9.0</version>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef787200/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 242f355..7432d2f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,6 +112,7 @@
<snappy.version>1.1.2.6</snappy.version>
<hadoop.version>2.7.2</hadoop.version>
<httpclient.version>4.2.5</httpclient.version>
+ <httpcore.version>${httpclient.version}</httpcore.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11.8</scala.version>
<hadoop.deps.scope>compile</hadoop.deps.scope>
[12/21] carbondata git commit: [CARBONDATA-2563][CATALYST] Explain
query with Order by operator is fired Spark Job which is increase explain
query time even though isQueryStatisticsEnabled is false
Posted by ra...@apache.org.
[CARBONDATA-2563][CATALYST] Explain query with Order by operator is fired Spark Job
which is increase explain query time even though isQueryStatisticsEnabled is false
Even though isQueryStatisticsEnabled is false which means user doesnt wants to see
statistics for explain command, still the engine tries to fetch the paritions information
which causes a job execution in case of order by query, this is mainly because
spark engine does sampling for defifning certain range within paritions for sorting process.
As part of solution the explain command process shall fetch the parition info only if isQueryStatisticsEnabled true.
This closes #2974
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/21762c54
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/21762c54
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/21762c54
Branch: refs/heads/branch-1.5
Commit: 21762c541df334816c660eb8e4e26aca1df9aa71
Parents: 93268ea
Author: s71955 <su...@gmail.com>
Authored: Tue Dec 4 20:48:55 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:33 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/profiler/ExplainCollector.java | 6 +++++-
.../execution/command/table/CarbonExplainCommand.scala | 11 ++++++++---
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21762c54/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
index 8513dac..75568e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
@@ -171,7 +171,11 @@ public class ExplainCollector {
}
public static String getFormatedOutput() {
- return get().toString();
+ if (null != get()) {
+ return get().toString();
+ } else {
+ return null;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21762c54/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
index 8939c6a..aa7a541 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonExplainCommand.scala
@@ -47,12 +47,17 @@ case class CarbonExplainCommand(
}
private def collectProfiler(sparkSession: SparkSession): Seq[Row] = {
- val queryExecution =
- sparkSession.sessionState.executePlan(child.asInstanceOf[ExplainCommand].logicalPlan)
try {
ExplainCollector.setup()
- queryExecution.toRdd.partitions
if (ExplainCollector.enabled()) {
+ val queryExecution =
+ sparkSession.sessionState.executePlan(child.asInstanceOf[ExplainCommand].logicalPlan)
+ queryExecution.toRdd.partitions
+ // For count(*) queries the explain collector will be disabled, so profiler
+ // informations not required in such scenarios.
+ if (null == ExplainCollector.getFormatedOutput) {
+ Seq.empty
+ }
Seq(Row("== CarbonData Profiler ==\n" + ExplainCollector.getFormatedOutput))
} else {
Seq.empty
[16/21] carbondata git commit: [CARBONDATA-3002] Fix some spell error
Posted by ra...@apache.org.
[CARBONDATA-3002] Fix some spell error
Fix some spell error
This closes #2890
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/313ba0ea
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/313ba0ea
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/313ba0ea
Branch: refs/heads/branch-1.5
Commit: 313ba0ea63f90f34f1b27f8940030c2999bd818c
Parents: c9b136c
Author: xubo245 <xu...@huawei.com>
Authored: Thu Nov 1 11:16:43 2018 +0800
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:34 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/datastore/FileReader.java | 6 ++--
.../core/datastore/impl/FileReaderImpl.java | 8 ++---
.../carbondata/core/locks/CarbonLockUtil.java | 4 +--
.../AbstractDetailQueryResultIterator.java | 8 ++---
.../carbondata/core/util/CarbonProperties.java | 8 ++---
.../sdv/register/TestRegisterCarbonTable.scala | 20 +++++------
...ithColumnMetCacheAndCacheLevelProperty.scala | 4 +--
.../createTable/TestCreateTableAsSelect.scala | 2 +-
.../DBLocationCarbonTableTestCase.scala | 4 +--
.../iud/DeleteCarbonTableTestCase.scala | 6 ++--
.../StandardPartitionTableLoadingTestCase.scala | 2 +-
.../apache/spark/rdd/CarbonMergeFilesRDD.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 4 +--
.../sql/CarbonDatasourceHadoopRelation.scala | 2 +-
.../scala/org/apache/spark/sql/CarbonEnv.scala | 12 +++----
.../org/apache/spark/sql/CarbonSource.scala | 8 ++---
.../datamap/CarbonDropDataMapCommand.scala | 2 +-
.../CarbonAlterTableCompactionCommand.scala | 2 +-
.../management/CarbonLoadDataCommand.scala | 2 +-
.../management/RefreshCarbonTableCommand.scala | 2 +-
.../spark/sql/execution/command/package.scala | 2 +-
.../CarbonAlterTableDropPartitionCommand.scala | 2 +-
.../CarbonAlterTableSplitPartitionCommand.scala | 2 +-
.../CarbonShowCarbonPartitionsCommand.scala | 2 +-
.../preaaggregate/PreAggregateUtil.scala | 8 ++---
.../CarbonAlterTableAddColumnCommand.scala | 2 +-
.../CarbonAlterTableDataTypeChangeCommand.scala | 2 +-
.../CarbonAlterTableDropColumnCommand.scala | 2 +-
.../schema/CarbonAlterTableRenameCommand.scala | 2 +-
.../CarbonCreateTableAsSelectCommand.scala | 2 +-
.../table/CarbonCreateTableCommand.scala | 4 +--
.../table/CarbonDescribeFormattedCommand.scala | 2 +-
.../command/table/CarbonDropTableCommand.scala | 2 +-
.../sql/execution/strategy/DDLStrategy.scala | 36 ++++++++++----------
.../spark/sql/hive/CarbonFileMetastore.scala | 2 +-
.../sql/hive/CarbonPreAggregateRules.scala | 4 +--
.../sql/parser/CarbonSparkSqlParserUtil.scala | 2 +-
.../org/apache/spark/util/AlterTableUtil.scala | 16 ++++-----
.../org/apache/spark/util/CleanFiles.scala | 2 +-
.../org/apache/spark/util/Compaction.scala | 2 +-
.../apache/spark/util/DeleteSegmentByDate.scala | 2 +-
.../apache/spark/util/DeleteSegmentById.scala | 2 +-
.../org/apache/spark/util/TableAPIUtil.scala | 2 +-
.../org/apache/spark/util/TableLoader.scala | 2 +-
.../spark/sql/hive/CarbonSessionState.scala | 4 +--
.../spark/util/AllDictionaryTestCase.scala | 2 +-
.../spark/util/DictionaryLRUCacheTestCase.scala | 2 +-
.../util/ExternalColumnDictionaryTestCase.scala | 2 +-
.../carbondata/TestStreamingTableOpName.scala | 26 +++++++-------
.../TestStreamingTableWithRowParser.scala | 2 +-
.../register/TestRegisterCarbonTable.scala | 22 ++++++------
.../loading/CarbonDataLoadConfiguration.java | 2 +-
.../loading/model/CarbonLoadModel.java | 2 +-
.../processing/util/CarbonLoaderUtil.java | 14 ++++----
54 files changed, 146 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
index df0d745..2527f1d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
@@ -56,7 +56,7 @@ public interface FileReader {
/**
* This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
+ * length will be always 4 because int byte size if 4
*
* @param filePath fully qualified file path
* @param offset reading start position,
@@ -66,7 +66,7 @@ public interface FileReader {
/**
* This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
+ * length will be always 8 because int byte size is 8
*
* @param filePath fully qualified file path
* @param offset reading start position,
@@ -76,7 +76,7 @@ public interface FileReader {
/**
* This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
+ * length will be always 4 because int byte size if 4
*
* @param filePath fully qualified file path
* @return read int
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
index 0c1b2b0..f513cb9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
@@ -81,7 +81,7 @@ public class FileReaderImpl implements FileReader {
/**
* This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
+ * length will be always 4 because int byte size if 4
*
* @param filePath fully qualified file path
* @param offset reading start position,
@@ -95,7 +95,7 @@ public class FileReaderImpl implements FileReader {
/**
* This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
+ * length will be always 4 because int byte size if 4
*
* @param filePath fully qualified file path
* @return read int
@@ -108,7 +108,7 @@ public class FileReaderImpl implements FileReader {
/**
* This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
+ * length will be always 4 because int byte size if 4
*
* @param filePath fully qualified file path
* @param offset reading start position,
@@ -184,7 +184,7 @@ public class FileReaderImpl implements FileReader {
/**
* This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
+ * length will be always 8 because int byte size is 8
*
* @param filePath fully qualified file path
* @param offset reading start position,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
index 89ccbd0..fbbf0c6 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
@@ -121,7 +121,7 @@ public class CarbonLockUtil {
*/
public static void deleteExpiredSegmentLockFiles(CarbonTable carbonTable) {
final long currTime = System.currentTimeMillis();
- final long segmentLockFilesPreservTime =
+ final long segmentLockFilesPreserveTime =
CarbonProperties.getInstance().getSegmentLockFilesPreserveHours();
AbsoluteTableIdentifier absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
String lockFilesDir = CarbonProperties.getInstance()
@@ -137,7 +137,7 @@ public class CarbonLockUtil {
@Override public boolean accept(CarbonFile pathName) {
if (CarbonTablePath.isSegmentLockFilePath(pathName.getName())) {
- return (currTime - pathName.getLastModifiedTime()) > segmentLockFilesPreservTime;
+ return (currTime - pathName.getLastModifiedTime()) > segmentLockFilesPreserveTime;
}
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index ed78aa7..9282d44 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -107,11 +107,11 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getTablePath()));
this.fileReader.setReadPageByPage(queryModel.isReadPageByPage());
this.execService = execService;
- intialiseInfos();
+ initialiseInfos();
initQueryStatiticsModel();
}
- private void intialiseInfos() {
+ private void initialiseInfos() {
for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
Map<String, DeleteDeltaVo> deletedRowsMap = null;
// if delete delta file is present
@@ -172,7 +172,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
carbonDeleteDeltaFileReader = new CarbonDeleteFilesDataReader();
Map<String, DeleteDeltaVo> deletedRowsMap = carbonDeleteDeltaFileReader
.getDeletedRowsDataVo(deleteDeltaInfo.getDeleteDeltaFile());
- setDeltedDeltaBoToDataBlock(deleteDeltaInfo, deletedRowsMap, dataBlock);
+ setDeletedDeltaBoToDataBlock(deleteDeltaInfo, deletedRowsMap, dataBlock);
// remove the lock
deleteDeltaToLockObjectMap.remove(deleteDeltaInfo);
return deletedRowsMap;
@@ -193,7 +193,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
* @param deletedRecordsMap
* @param dataBlock
*/
- private void setDeltedDeltaBoToDataBlock(DeleteDeltaInfo deleteDeltaInfo,
+ private void setDeletedDeltaBoToDataBlock(DeleteDeltaInfo deleteDeltaInfo,
Map<String, DeleteDeltaVo> deletedRecordsMap, AbstractIndex dataBlock) {
// check if timestamp of data block is less than the latest delete delta timestamp
// then update the delete delta details and timestamp in data block
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index f4a75a8..93d622d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -593,21 +593,21 @@ public final class CarbonProperties {
* This method validates the number of column read in one IO
*/
private void validateNumberOfColumnPerIORead() {
- String numberofColumnPerIOString = carbonProperties
+ String numberOfColumnPerIOString = carbonProperties
.getProperty(NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE);
try {
- short numberofColumnPerIO = Short.parseShort(numberofColumnPerIOString);
+ short numberofColumnPerIO = Short.parseShort(numberOfColumnPerIOString);
if (numberofColumnPerIO < CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_MIN
|| numberofColumnPerIO > CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_MAX) {
- LOGGER.info("The Number Of pages per blocklet column value \"" + numberofColumnPerIOString
+ LOGGER.info("The Number Of pages per blocklet column value \"" + numberOfColumnPerIOString
+ "\" is invalid. Using the default value \""
+ CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE);
carbonProperties.setProperty(NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE);
}
} catch (NumberFormatException e) {
- LOGGER.info("The Number Of pages per blocklet column value \"" + numberofColumnPerIOString
+ LOGGER.info("The Number Of pages per blocklet column value \"" + numberOfColumnPerIOString
+ "\" is invalid. Using the default value \""
+ CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE);
carbonProperties.setProperty(NUMBER_OF_COLUMN_TO_READ_IN_IO,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
index bf07bd6..caae8e1 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/register/TestRegisterCarbonTable.scala
@@ -83,7 +83,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("use carbon")
sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql("insert into carbontable select 'a',1,'aa','aaa'")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
sql("drop table carbontable")
restoreData(dbLocationCustom, "carbontable")
@@ -99,7 +99,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("use carbon")
sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql("insert into carbontable select 'a',1,'aa','aaa'")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
sql("drop table carbontable")
restoreData(dbLocationCustom, "carbontable")
@@ -118,7 +118,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'b',1,'aa','aaa'")
sql("insert into carbontable select 'a',10,'aa','aaa'")
sql("create datamap preagg1 on table carbontable using 'preaggregate' as select c1,sum(c2) from carbontable group by c1")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
backUpData(dbLocationCustom, "carbontable_preagg1")
sql("drop table carbontable")
@@ -141,7 +141,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'b',1,'aa','aaa'")
sql("insert into carbontable select 'a',10,'aa','aaa'")
sql("create datamap preagg1 on table carbontable using 'preaggregate' as select c1,sum(c2) from carbontable group by c1")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
backUpData(dbLocationCustom, "carbontable_preagg1")
sql("drop table carbontable")
@@ -164,7 +164,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'b',1,'aa','aaa'")
sql("insert into carbontable select 'a',10,'aa','aaa'")
sql("create datamap preagg1 on table carbontable using 'preaggregate' as select c1,sum(c2) from carbontable group by c1")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
backUpData(dbLocationCustom, "carbontable_preagg1")
sql("drop table carbontable")
@@ -183,7 +183,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql("insert into carbontable select 'a',1,'aa','aaa'")
sql("insert into carbontable select 'b',1,'bb','bbb'")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
sql("drop table carbontable")
restoreData(dbLocationCustom, "carbontable")
@@ -205,7 +205,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql("insert into carbontable select 'a',1,'aa','aaa'")
sql("insert into carbontable select 'b',1,'bb','bbb'")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
sql("drop table carbontable")
restoreData(dbLocationCustom, "carbontable")
@@ -227,7 +227,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql("insert into carbontable select 'a',1,'aa','aaa'")
sql("insert into carbontable select 'b',1,'bb','bbb'")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
sql("drop table carbontable")
restoreData(dbLocationCustom, "carbontable")
@@ -249,7 +249,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql("insert into carbontable select 'a',1,'aa','aaa'")
sql("insert into carbontable select 'b',1,'bb','bbb'")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
sql("drop table carbontable")
restoreData(dbLocationCustom, "carbontable")
@@ -270,7 +270,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("""create table carbon.carbontable (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql("insert into carbontable select 'a',1,'aa','aaa'")
sql("insert into carbontable select 'b',1,'bb','bbb'")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
backUpData(dbLocationCustom, "carbontable")
sql("drop table carbontable")
restoreData(dbLocationCustom, "carbontable")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithColumnMetCacheAndCacheLevelProperty.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithColumnMetCacheAndCacheLevelProperty.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithColumnMetCacheAndCacheLevelProperty.scala
index 1c54c48..7c9a9fc 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithColumnMetCacheAndCacheLevelProperty.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithColumnMetCacheAndCacheLevelProperty.scala
@@ -75,7 +75,7 @@ class TestQueryWithColumnMetCacheAndCacheLevelProperty extends QueryTest with Be
tableName: String,
segmentId: String,
isSchemaModified: Boolean = false): List[DataMap[_ <: Blocklet]] = {
- val relation: CarbonRelation = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
+ val relation: CarbonRelation = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore
.lookupRelation(Some(dbName), tableName)(sqlContext.sparkSession)
.asInstanceOf[CarbonRelation]
val carbonTable = relation.carbonTable
@@ -291,7 +291,7 @@ class TestQueryWithColumnMetCacheAndCacheLevelProperty extends QueryTest with Be
sql("insert into minMaxSerialize select 'a','aa','aaa'")
checkAnswer(sql("select * from minMaxSerialize where name='a'"), Row("a", "aa", "aaa"))
checkAnswer(sql("select * from minMaxSerialize where name='b'"), Seq.empty)
- val relation: CarbonRelation = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
+ val relation: CarbonRelation = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore
.lookupRelation(Some("default"), "minMaxSerialize")(sqlContext.sparkSession)
.asInstanceOf[CarbonRelation]
val carbonTable = relation.carbonTable
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
index c95e5a4..3896061 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateTableAsSelect.scala
@@ -151,7 +151,7 @@ class TestCreateTableAsSelect extends QueryTest with BeforeAndAfterAll {
"create table ctas_tblproperties_testt stored by 'carbondata' TBLPROPERTIES" +
"('DICTIONARY_INCLUDE'='key', 'sort_scope'='global_sort') as select * from carbon_ctas_test")
checkAnswer(sql("select * from ctas_tblproperties_testt"), sql("select * from carbon_ctas_test"))
- val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetastore
+ val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetaStore
.lookupRelation(Option("default"), "ctas_tblproperties_testt")(Spark2TestQueryExecutor.spark)
.asInstanceOf[CarbonRelation].carbonTable
val metadataFolderPath: CarbonFile = FileFactory.getCarbonFile(carbonTable.getMetadataPath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dblocation/DBLocationCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dblocation/DBLocationCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dblocation/DBLocationCarbonTableTestCase.scala
index cb47cf5..7b80c72 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dblocation/DBLocationCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dblocation/DBLocationCarbonTableTestCase.scala
@@ -284,7 +284,7 @@ class DBLocationCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table carbontable")
// perform file check
assert(FileFactory.isFileExist(timestampFile, timestampFileType, true) ||
- CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore)
+ CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore)
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_UPDATE_SYNC_FOLDER,
@@ -295,7 +295,7 @@ class DBLocationCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table carbontable")
// perform file check
assert(FileFactory.isFileExist(timestampFile, timestampFileType, true) ||
- CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore)
+ CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore)
}
override def afterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index 39582f0..2f95133 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -208,7 +208,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
.getCarbonTable(Some("iud_db"), "update_status_files")(sqlContext.sparkSession)
val metaPath = carbonTable.getMetadataPath
val files = FileFactory.getCarbonFile(metaPath)
- val result = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.getClass
+ val result = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.getClass
if(result.getCanonicalName.contains("CarbonFileMetastore")) {
assert(files.listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = !file.isDirectory
@@ -257,11 +257,11 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
assert(
listOfTupleId(4).contains("0/0/0-0_batchno0-0-0-") && listOfTupleId(4).endsWith("/0/0/4"))
- val carbonTable_part = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetastore
+ val carbonTable_part = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetaStore
.lookupRelation(Option("iud_db"), "dest_tuple_part")(Spark2TestQueryExecutor.spark)
.asInstanceOf[CarbonRelation].carbonTable
- val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetastore
+ val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetaStore
.lookupRelation(Option("iud_db"), "dest_tuple")(Spark2TestQueryExecutor.spark)
.asInstanceOf[CarbonRelation].carbonTable
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 9a0080c..059dd2b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -406,7 +406,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
val dblocation = table.getTablePath.substring(0, table.getTablePath.lastIndexOf("/"))
backUpData(dblocation, "restorepartition")
sql("drop table restorepartition")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "restorepartition")
sql("refresh table restorepartition")
checkAnswer(sql("select count(*) from restorepartition"), rows)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark-common/src/main/scala/org/apache/spark/rdd/CarbonMergeFilesRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/rdd/CarbonMergeFilesRDD.scala b/integration/spark-common/src/main/scala/org/apache/spark/rdd/CarbonMergeFilesRDD.scala
index 3605dde..c101d02 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/rdd/CarbonMergeFilesRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/rdd/CarbonMergeFilesRDD.scala
@@ -127,7 +127,7 @@ class CarbonMergeFilesRDD(
if (isHivePartitionedTable) {
CarbonLoaderUtil
- .mergeIndexFilesinPartitionedSegment(carbonTable, split.segmentId,
+ .mergeIndexFilesInPartitionedSegment(carbonTable, split.segmentId,
segmentFileNameToSegmentIdMap.get(split.segmentId))
} else {
new CarbonIndexFileMergeWriter(carbonTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index a849c99..34c6592 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -192,7 +192,7 @@ object CarbonDataRDDFactory {
LOGGER.info("System level compaction lock is enabled.")
val skipCompactionTables = ListBuffer[CarbonTableIdentifier]()
var tableForCompaction = CarbonCompactionUtil.getNextTableToCompact(
- CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
+ CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore
.listAllTables(sqlContext.sparkSession).toArray,
skipCompactionTables.toList.asJava)
while (null != tableForCompaction) {
@@ -247,7 +247,7 @@ object CarbonDataRDDFactory {
}
// ********* check again for all the tables.
tableForCompaction = CarbonCompactionUtil.getNextTableToCompact(
- CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
+ CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore
.listAllTables(sqlContext.sparkSession).toArray,
skipCompactionTables.asJava)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 04ec75d..f848ae1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -61,7 +61,7 @@ case class CarbonDatasourceHadoopRelation(
CarbonSession.updateSessionInfoToCurrentThread(sparkSession)
@transient lazy val carbonRelation: CarbonRelation =
- CarbonEnv.getInstance(sparkSession).carbonMetastore.
+ CarbonEnv.getInstance(sparkSession).carbonMetaStore.
createCarbonRelation(parameters, identifier, sparkSession)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 90ba58c..e114e06 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -45,7 +45,7 @@ import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
*/
class CarbonEnv {
- var carbonMetastore: CarbonMetaStore = _
+ var carbonMetaStore: CarbonMetaStore = _
var sessionParams: SessionParams = _
@@ -53,7 +53,7 @@ class CarbonEnv {
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- // set readsupport class global so that the executor can get it.
+ // set readSupport class global so that the executor can get it.
SparkReadSupport.readSupportClass = classOf[SparkRowReadSupportImpl]
var initialized = false
@@ -105,7 +105,7 @@ class CarbonEnv {
}
// add session params after adding DefaultCarbonParams
config.addDefaultCarbonSessionParams()
- carbonMetastore = {
+ carbonMetaStore = {
// trigger event for CarbonEnv create
val operationContext = new OperationContext
val carbonEnvInitPreEvent: CarbonEnvInitPreEvent =
@@ -195,7 +195,7 @@ object CarbonEnv {
(sparkSession: SparkSession): CarbonTable = {
refreshRelationFromCache(TableIdentifier(tableName, databaseNameOp))(sparkSession)
val databaseName = getDatabaseName(databaseNameOp)(sparkSession)
- val catalog = getInstance(sparkSession).carbonMetastore
+ val catalog = getInstance(sparkSession).carbonMetaStore
// refresh cache
catalog.checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, databaseNameOp))
@@ -211,10 +211,10 @@ object CarbonEnv {
def refreshRelationFromCache(identifier: TableIdentifier)(sparkSession: SparkSession): Boolean = {
var isRefreshed = false
val carbonEnv = getInstance(sparkSession)
- val table = carbonEnv.carbonMetastore.getTableFromMetadataCache(
+ val table = carbonEnv.carbonMetaStore.getTableFromMetadataCache(
identifier.database.getOrElse(sparkSession.sessionState.catalog.getCurrentDatabase),
identifier.table)
- if (carbonEnv.carbonMetastore
+ if (carbonEnv.carbonMetaStore
.checkSchemasModifiedTimeAndReloadTable(identifier) && table.isDefined) {
sparkSession.sessionState.catalog.refreshTable(identifier)
val tablePath = table.get.getTablePath
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index cd1087d..9899e8b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -91,7 +91,7 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
CarbonEnv.getInstance(sqlContext.sparkSession)
var newParameters = CarbonScalaUtil.getDeserializedParameters(parameters)
val options = new CarbonOption(newParameters)
- val isExists = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.tableExists(
+ val isExists = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.tableExists(
options.tableName, options.dbName)(sqlContext.sparkSession)
val (doSave, doAppend) = (mode, isExists) match {
case (SaveMode.ErrorIfExists, true) =>
@@ -182,7 +182,7 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
case _: NoSuchTableException =>
LOGGER.warn("Carbon Table [" +dbName +"] [" +tableName +"] is not found, " +
"Now existing Schema will be overwritten with default properties")
- val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val identifier = AbsoluteTableIdentifier.from(
CarbonEnv.getTablePath(Some(dbName), tableName)(sparkSession),
dbName,
@@ -295,7 +295,7 @@ object CarbonSource {
"Schema cannot be specified in a Create Table As Select (CTAS) statement")
}
sqlParser
- .getFields(CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .getFields(CarbonEnv.getInstance(sparkSession).carbonMetaStore
.getSchemaFromUnresolvedRelation(sparkSession, q))
case None =>
sqlParser.getFields(dataSchema)
@@ -315,7 +315,7 @@ object CarbonSource {
tableDesc: CatalogTable,
sparkSession: SparkSession,
query: Option[LogicalPlan] = None): CatalogTable = {
- val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val storageFormat = tableDesc.storage
val properties = storageFormat.properties
if (!properties.contains("carbonSchemaPartsNo")) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 38ec07d..54096ca 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -65,7 +65,7 @@ case class CarbonDropDataMapCommand(
val dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
val locksToBeAcquired = List(LockUsage.METADATA_LOCK)
val carbonEnv = CarbonEnv.getInstance(sparkSession)
- val catalog = carbonEnv.carbonMetastore
+ val catalog = carbonEnv.carbonMetaStore
val tablePath = CarbonEnv.getTablePath(databaseNameOp, tableName)(sparkSession)
catalog.checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
if (mainTable == null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index 6edfcf4..a908a84 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -74,7 +74,7 @@ case class CarbonAlterTableCompactionCommand(
table = if (tableInfoOp.isDefined) {
CarbonTable.buildFromTableInfo(tableInfoOp.get)
} else {
- val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val relation = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
if (relation == null) {
throw new NoSuchTableException(dbName, tableName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 7a974e3..3d2924c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -114,7 +114,7 @@ case class CarbonLoadDataCommand(
table = if (tableInfoOp.isDefined) {
CarbonTable.buildFromTableInfo(tableInfoOp.get)
} else {
- val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val relation = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
if (relation == null) {
throw new NoSuchTableException(dbName, tableName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
index b35c285..6a9ac0a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/RefreshCarbonTableCommand.scala
@@ -51,7 +51,7 @@ case class RefreshCarbonTableCommand(
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
- val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val databaseName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
setAuditTable(databaseName, tableName)
// Steps
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/package.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/package.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/package.scala
index f7f76b9..8073f90 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/package.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/package.scala
@@ -36,7 +36,7 @@ object Checker {
session: SparkSession): Unit = {
val database = dbName.getOrElse(session.catalog.currentDatabase)
val identifier = TableIdentifier(tableName, dbName)
- if (!CarbonEnv.getInstance(session).carbonMetastore.tableExists(identifier)(session)) {
+ if (!CarbonEnv.getInstance(session).carbonMetaStore.tableExists(identifier)(session)) {
val err = s"table $dbName.$tableName not found"
LogServiceFactory.getLogService(this.getClass.getName).error(err)
throw new NoSuchTableException(database, tableName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
index 832cb00..b40bb6d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
@@ -59,7 +59,7 @@ case class CarbonAlterTableDropPartitionCommand(
val tableName = model.tableName
setAuditTable(dbName, tableName)
setAuditInfo(Map("partition" -> model.partitionId))
- val carbonMetaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val carbonMetaStore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val relation = carbonMetaStore.lookupRelation(Option(dbName), tableName)(sparkSession)
.asInstanceOf[CarbonRelation]
val tablePath = relation.carbonTable.getTablePath
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
index f17cdd6..4d32d00 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
@@ -59,7 +59,7 @@ case class CarbonAlterTableSplitPartitionCommand(
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val dbName = splitPartitionModel.databaseName.getOrElse(sparkSession.catalog.currentDatabase)
- val carbonMetaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val carbonMetaStore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val tableName = splitPartitionModel.tableName
setAuditTable(dbName, tableName)
setAuditInfo(Map("partition" -> splitPartitionModel.partitionId))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonShowCarbonPartitionsCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonShowCarbonPartitionsCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonShowCarbonPartitionsCommand.scala
index 2915981..0e57513 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonShowCarbonPartitionsCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonShowCarbonPartitionsCommand.scala
@@ -36,7 +36,7 @@ private[sql] case class CarbonShowCarbonPartitionsCommand(
override val output: Seq[Attribute] = CommonUtil.partitionInfoOutput
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
- val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val relation = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.lookupRelation(tableIdentifier)(sparkSession).asInstanceOf[CarbonRelation]
val carbonTable = relation.carbonTable
setAuditTable(carbonTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index 0599fb3..319f84b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -426,7 +426,7 @@ object PreAggregateUtil {
val dbName = carbonTable.getDatabaseName
val tableName = carbonTable.getTableName
try {
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
locks = acquireLock(dbName, tableName, locksToBeAcquired, carbonTable)
// get the latest carbon table and check for column existence
// read the latest schema file
@@ -468,7 +468,7 @@ object PreAggregateUtil {
thriftTable: TableInfo)(sparkSession: SparkSession): Unit = {
val dbName = carbonTable.getDatabaseName
val tableName = carbonTable.getTableName
- CarbonEnv.getInstance(sparkSession).carbonMetastore
+ CarbonEnv.getInstance(sparkSession).carbonMetaStore
.updateTableSchemaForDataMap(carbonTable.getCarbonTableIdentifier,
carbonTable.getCarbonTableIdentifier,
thriftTable,
@@ -527,7 +527,7 @@ object PreAggregateUtil {
*/
def revertMainTableChanges(dbName: String, tableName: String, numberOfChildSchema: Int)
(sparkSession: SparkSession): Unit = {
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
carbonTable.getTableLastUpdatedTime
val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)
@@ -539,7 +539,7 @@ object PreAggregateUtil {
def getChildCarbonTable(databaseName: String, tableName: String)
(sparkSession: SparkSession): Option[CarbonTable] = {
- val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val carbonTable = metaStore.getTableFromMetadataCache(databaseName, tableName)
if (carbonTable.isEmpty) {
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
index 719ed4a..9c92614 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
@@ -57,7 +57,7 @@ private[sql] case class CarbonAlterTableAddColumnCommand(
// completion of 1st operation but as look up relation is called before it will have the
// older carbon table and this can lead to inconsistent state in the system. Therefor look
// up relation should be called after acquiring the lock
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_ADD_COLUMN)) {
throw new MalformedCarbonCommandException(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
index 2bcd3aa..23a7615 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
@@ -56,7 +56,7 @@ private[sql] case class CarbonAlterTableDataTypeChangeCommand(
try {
locks = AlterTableUtil
.validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_CHANGE_DATATYPE,
alterTableDataTypeChangeModel.columnName)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
index ccf9e54..9ef6fd8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
@@ -55,7 +55,7 @@ private[sql] case class CarbonAlterTableDropColumnCommand(
try {
locks = AlterTableUtil
.validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_DROP,
alterTableDropColumnModel.columns.asJava)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index c64f50b..dbf665a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -62,7 +62,7 @@ private[sql] case class CarbonAlterTableRenameCommand(
val oldTableName = oldTableIdentifier.table.toLowerCase
val newTableName = newTableIdentifier.table.toLowerCase
LOGGER.info(s"Rename table request has been received for $oldDatabaseName.$oldTableName")
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val relation: CarbonRelation =
metastore.lookupRelation(oldTableIdentifier.database, oldTableName)(sparkSession)
.asInstanceOf[CarbonRelation]
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
index 54be619..a5889ca 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
@@ -72,7 +72,7 @@ case class CarbonCreateTableAsSelectCommand(
databaseOpt = Some(tableInfo.getDatabaseName)
}
val dbName = CarbonEnv.getDatabaseName(databaseOpt)(sparkSession)
- val carbonDataSourceHadoopRelation = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val carbonDataSourceHadoopRelation = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.createCarbonDataSourceHadoopRelation(sparkSession,
TableIdentifier(tableName, Option(dbName)))
// execute command to load data into carbon table
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index 713561b..a13e8e0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -115,7 +115,7 @@ case class CarbonCreateTableCommand(
val createTablePreExecutionEvent: CreateTablePreExecutionEvent =
CreateTablePreExecutionEvent(sparkSession, tableIdentifier, Some(tableInfo))
OperationListenerBus.getInstance.fireEvent(createTablePreExecutionEvent, operationContext)
- val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val catalog = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val carbonSchemaString = catalog.generateTableSchemaString(tableInfo, tableIdentifier)
if (createDSTable) {
try {
@@ -170,7 +170,7 @@ case class CarbonCreateTableCommand(
case e: Exception =>
// call the drop table to delete the created table.
try {
- CarbonEnv.getInstance(sparkSession).carbonMetastore
+ CarbonEnv.getInstance(sparkSession).carbonMetaStore
.dropTable(tableIdentifier)(sparkSession)
} catch {
case _: Exception => // No operation
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index 2d560df..69db4e0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -44,7 +44,7 @@ private[sql] case class CarbonDescribeFormattedCommand(
extends MetadataCommand {
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
- val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val relation = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.lookupRelation(tblIdentifier)(sparkSession).asInstanceOf[CarbonRelation]
setAuditTable(relation.databaseName, relation.tableName)
var results: Seq[(String, String, String)] = child.schema.fields.map { field =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
index 0505a75..f69ef9e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
@@ -95,7 +95,7 @@ case class CarbonDropTableCommand(
sparkSession)
OperationListenerBus.getInstance.fireEvent(dropTablePreEvent, operationContext)
- CarbonEnv.getInstance(sparkSession).carbonMetastore.dropTable(identifier)(sparkSession)
+ CarbonEnv.getInstance(sparkSession).carbonMetaStore.dropTable(identifier)(sparkSession)
if (carbonTable.hasDataMapSchema) {
// drop all child tables
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 4cc62c6..3c9e538 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -58,7 +58,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
def apply(plan: LogicalPlan): Seq[SparkPlan] = {
plan match {
case LoadDataCommand(identifier, path, isLocal, isOverwrite, partition)
- if CarbonEnv.getInstance(sparkSession).carbonMetastore
+ if CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(identifier)(sparkSession) =>
ExecutedCommandExec(
CarbonLoadDataCommand(
@@ -78,7 +78,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
case alter@AlterTableRenameCommand(oldTableIdentifier, newTableIdentifier, _) =>
val dbOption = oldTableIdentifier.database.map(_.toLowerCase)
val tableIdentifier = TableIdentifier(oldTableIdentifier.table.toLowerCase(), dbOption)
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(tableIdentifier)(sparkSession)
if (isCarbonTable) {
val renameModel = AlterTableRenameModel(tableIdentifier, newTableIdentifier)
@@ -87,13 +87,13 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
ExecutedCommandExec(alter) :: Nil
}
case DropTableCommand(identifier, ifNotExists, isView, _)
- if CarbonEnv.getInstance(sparkSession).carbonMetastore
+ if CarbonEnv.getInstance(sparkSession).carbonMetaStore
.isTablePathExists(identifier)(sparkSession) =>
ExecutedCommandExec(
CarbonDropTableCommand(ifNotExists, identifier.database,
identifier.table.toLowerCase)) :: Nil
case createLikeTable: CreateTableLikeCommand =>
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(createLikeTable.sourceTable)(sparkSession)
if (isCarbonTable) {
throw new MalformedCarbonCommandException(
@@ -118,7 +118,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
case drop@DropDatabaseCommand(dbName, ifExists, isCascade) =>
ExecutedCommandExec(CarbonDropDatabaseCommand(drop)) :: Nil
case alterTable@CarbonAlterTableCompactionCommand(altertablemodel, _, _) =>
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(TableIdentifier(altertablemodel.tableName,
altertablemodel.dbName))(sparkSession)
if (isCarbonTable) {
@@ -128,7 +128,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
"Operation not allowed : " + altertablemodel.alterSql)
}
case dataTypeChange@CarbonAlterTableDataTypeChangeCommand(alterTableChangeDataTypeModel) =>
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(TableIdentifier(alterTableChangeDataTypeModel.tableName,
alterTableChangeDataTypeModel.databaseName))(sparkSession)
if (isCarbonTable) {
@@ -147,7 +147,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
throw new MalformedCarbonCommandException("Unsupported alter operation on hive table")
}
case addColumn@CarbonAlterTableAddColumnCommand(alterTableAddColumnsModel) =>
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(TableIdentifier(alterTableAddColumnsModel.tableName,
alterTableAddColumnsModel.databaseName))(sparkSession)
if (isCarbonTable) {
@@ -181,7 +181,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
throw new MalformedCarbonCommandException("Unsupported alter operation on hive table")
}
case dropColumn@CarbonAlterTableDropColumnCommand(alterTableDropColumnModel) =>
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(TableIdentifier(alterTableDropColumnModel.tableName,
alterTableDropColumnModel.databaseName))(sparkSession)
if (isCarbonTable) {
@@ -206,7 +206,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
} else {
false
}
- if (CarbonEnv.getInstance(sparkSession).carbonMetastore
+ if (CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(identifier)(sparkSession) && (isExtended || isFormatted)) {
val resolvedTable =
sparkSession.sessionState.executePlan(UnresolvedRelation(identifier)).analyzed
@@ -221,10 +221,10 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
Nil
}
case ShowPartitionsCommand(t, cols) =>
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(t)(sparkSession)
if (isCarbonTable) {
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.lookupRelation(t)(sparkSession).asInstanceOf[CarbonRelation].carbonTable
if (carbonTable != null && !carbonTable.getTableInfo.isTransactionalTable) {
throw new MalformedCarbonCommandException(
@@ -239,7 +239,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
ExecutedCommandExec(ShowPartitionsCommand(t, cols)) :: Nil
}
case adp@AlterTableDropPartitionCommand(tableName, specs, ifExists, purge, retainData) =>
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(tableName)(sparkSession)
if (isCarbonTable) {
ExecutedCommandExec(
@@ -290,10 +290,10 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
val cmd = CreateDataSourceTableCommand(updatedCatalog, ignoreIfExists)
ExecutedCommandExec(cmd) :: Nil
case AlterTableSetPropertiesCommand(tableName, properties, isView)
- if CarbonEnv.getInstance(sparkSession).carbonMetastore
+ if CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(tableName)(sparkSession) => {
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.lookupRelation(tableName)(sparkSession).asInstanceOf[CarbonRelation].carbonTable
if (carbonTable != null && !carbonTable.getTableInfo.isTransactionalTable) {
throw new MalformedCarbonCommandException(
@@ -323,7 +323,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
ExecutedCommandExec(CarbonAlterTableSetCommand(tableName, properties, isView)) :: Nil
}
case AlterTableUnsetPropertiesCommand(tableName, propKeys, ifExists, isView)
- if CarbonEnv.getInstance(sparkSession).carbonMetastore
+ if CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(tableName)(sparkSession) => {
// TODO remove this limitation later
if (propKeys.exists(_.equalsIgnoreCase("streaming"))) {
@@ -336,7 +336,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
case rename@AlterTableRenamePartitionCommand(tableName, oldPartition, newPartition) =>
val dbOption = tableName.database.map(_.toLowerCase)
val tableIdentifier = TableIdentifier(tableName.table.toLowerCase(), dbOption)
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(tableIdentifier)(sparkSession)
if (isCarbonTable) {
throw new UnsupportedOperationException("Renaming partition on table is not supported")
@@ -346,7 +346,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
case addPart@AlterTableAddPartitionCommand(tableName, partitionSpecsAndLocs, ifNotExists) =>
val dbOption = tableName.database.map(_.toLowerCase)
val tableIdentifier = TableIdentifier(tableName.table.toLowerCase(), dbOption)
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(tableIdentifier)(sparkSession)
if (isCarbonTable) {
ExecutedCommandExec(
@@ -371,7 +371,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
}
ExecutedCommandExec(plan.asInstanceOf[RunnableCommand]) :: Nil
case alterSetLoc@AlterTableSetLocationCommand(tableName, _, _) =>
- val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.tableExists(tableName)(sparkSession)
if (isCarbonTable) {
throw new UnsupportedOperationException("Set partition location is not supported")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index 96b31c2..c1be154 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -169,7 +169,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
}
val identifier: AbsoluteTableIdentifier = AbsoluteTableIdentifier.from(
catalogTable.location.toString, database, tableIdentifier.table)
- CarbonEnv.getInstance(sparkSession).carbonMetastore.
+ CarbonEnv.getInstance(sparkSession).carbonMetaStore.
createCarbonRelation(catalogTable.storage.properties, identifier, sparkSession)
case _ => throw new NoSuchTableException(database, tableIdentifier.table)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index 9b204f8..a8d7d22 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -738,7 +738,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
val identifier = TableIdentifier(
dataMapSchema.getChildSchema.getTableName,
Some(parentTable.getDatabaseName))
- val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val catalog = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val carbonRelation =
catalog.lookupRelation(identifier)(sparkSession).asInstanceOf[CarbonRelation]
val segmentStatusManager = new SegmentStatusManager(carbonRelation.carbonTable
@@ -1045,7 +1045,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
if (!selectedAggMaps.isEmpty) {
// filter the selected child schema based on size to select the pre-aggregate tables
// that are enabled
- val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val catalog = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val relationBuffer = selectedAggMaps.asScala.map { selectedDataMapSchema =>
val identifier = TableIdentifier(
selectedDataMapSchema.getRelationIdentifier.getTableName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
index 0378bf9..46473f2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
@@ -115,7 +115,7 @@ object CarbonSparkSqlParserUtil {
operationNotAllowed("Create external table as select", tableHeader)
}
fields = parser
- .getFields(CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .getFields(CarbonEnv.getInstance(sparkSession).carbonMetaStore
.getSchemaFromUnresolvedRelation(sparkSession, Some(q).get))
case _ =>
// ignore this case
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index 3faa111..756a1ad 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -63,7 +63,7 @@ object AlterTableUtil {
locksToBeAcquired: List[String])
(sparkSession: SparkSession): List[ICarbonLock] = {
val relation =
- CarbonEnv.getInstance(sparkSession).carbonMetastore
+ CarbonEnv.getInstance(sparkSession).carbonMetaStore
.lookupRelation(Option(dbName), tableName)(sparkSession)
.asInstanceOf[CarbonRelation]
if (relation == null) {
@@ -116,7 +116,7 @@ object AlterTableUtil {
Option[Seq[org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema]]) = {
val dbName = carbonTable.getDatabaseName
val tableName = carbonTable.getTableName
- CarbonEnv.getInstance(sparkSession).carbonMetastore
+ CarbonEnv.getInstance(sparkSession).carbonMetaStore
.updateTableSchemaForAlter(carbonTable.getCarbonTableIdentifier,
carbonTable.getCarbonTableIdentifier,
thriftTable,
@@ -124,7 +124,7 @@ object AlterTableUtil {
carbonTable.getAbsoluteTableIdentifier.getTablePath)(sparkSession)
val tableIdentifier = TableIdentifier(tableName, Some(dbName))
sparkSession.catalog.refreshTable(tableIdentifier.quotedString)
- val schema = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val schema = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.lookupRelation(tableIdentifier)(sparkSession).schema.json
val schemaParts = prepareSchemaJsonForAlterTable(sparkSession.sparkContext.getConf, schema)
(tableIdentifier, schemaParts, cols)
@@ -167,7 +167,7 @@ object AlterTableUtil {
val oldCarbonTableIdentifier = oldCarbonTable.getCarbonTableIdentifier
val database = oldCarbonTable.getDatabaseName
val newCarbonTableIdentifier = new CarbonTableIdentifier(database, newTableName, tableId)
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val fileType = FileFactory.getFileType(tablePath)
if (FileFactory.isFileExist(tablePath, fileType)) {
val tableInfo = metastore.getThriftTableInfo(oldCarbonTable)
@@ -195,7 +195,7 @@ object AlterTableUtil {
*/
def revertAddColumnChanges(dbName: String, tableName: String, timeStamp: Long)
(sparkSession: SparkSession): Unit = {
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
@@ -220,7 +220,7 @@ object AlterTableUtil {
*/
def revertDropColumnChanges(dbName: String, tableName: String, timeStamp: Long)
(sparkSession: SparkSession): Unit = {
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
@@ -251,7 +251,7 @@ object AlterTableUtil {
*/
def revertDataTypeChanges(dbName: String, tableName: String, timeStamp: Long)
(sparkSession: SparkSession): Unit = {
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
@@ -294,7 +294,7 @@ object AlterTableUtil {
try {
locks = AlterTableUtil
.validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetaStore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
val lowerCasePropertiesMap: mutable.Map[String, String] = mutable.Map.empty
// convert all the keys to lower case
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
index d4d9a84..cb3ae29 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
@@ -69,7 +69,7 @@ object CleanFiles {
forceTableClean = args(2).toBoolean
}
val spark = TableAPIUtil.spark(storePath, s"CleanFiles: $dbName.$tableName")
- CarbonEnv.getInstance(spark).carbonMetastore.
+ CarbonEnv.getInstance(spark).carbonMetaStore.
checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
cleanFiles(spark, dbName, tableName, forceTableClean)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala b/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala
index d4ec81e..0a3a870 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/Compaction.scala
@@ -58,7 +58,7 @@ object Compaction {
val (dbName, tableName) = TableAPIUtil.parseSchemaName(TableAPIUtil.escape(args(1)))
val compactionType = TableAPIUtil.escape(args(2))
val spark = TableAPIUtil.spark(storePath, s"Compaction: $dbName.$tableName")
- CarbonEnv.getInstance(spark).carbonMetastore.
+ CarbonEnv.getInstance(spark).carbonMetaStore.
checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
compaction(spark, dbName, tableName, compactionType)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
index fcc9f2f..90a37f6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
@@ -45,7 +45,7 @@ object DeleteSegmentByDate {
val (dbName, tableName) = TableAPIUtil.parseSchemaName(TableAPIUtil.escape(args(1)))
val dateValue = TableAPIUtil.escape(args(2))
val spark = TableAPIUtil.spark(storePath, s"DeleteSegmentByDate: $dbName.$tableName")
- CarbonEnv.getInstance(spark).carbonMetastore.
+ CarbonEnv.getInstance(spark).carbonMetaStore.
checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
deleteSegmentByDate(spark, dbName, tableName, dateValue)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
index 13ef933..15bec02 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
@@ -50,7 +50,7 @@ object DeleteSegmentById {
val (dbName, tableName) = TableAPIUtil.parseSchemaName(TableAPIUtil.escape(args(1)))
val segmentIds = extractSegmentIds(TableAPIUtil.escape(args(2)))
val spark = TableAPIUtil.spark(storePath, s"DeleteSegmentById: $dbName.$tableName")
- CarbonEnv.getInstance(spark).carbonMetastore.
+ CarbonEnv.getInstance(spark).carbonMetaStore.
checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
deleteSegmentById(spark, dbName, tableName, segmentIds)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
index a8094b6..0557e14 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
@@ -62,7 +62,7 @@ object TableAPIUtil {
spark: SparkSession,
dbName: String,
tableName: String): Unit = {
- if (!CarbonEnv.getInstance(spark).carbonMetastore
+ if (!CarbonEnv.getInstance(spark).carbonMetaStore
.tableExists(tableName, Some(dbName))(spark)) {
val err = s"table $dbName.$tableName not found"
LOGGER.error(err)
[11/21] carbondata git commit: [CARBONDATA-2755][Complex DataType
Enhancements] Compaction Complex Types (STRUCT AND ARRAY)
Posted by ra...@apache.org.
[CARBONDATA-2755][Complex DataType Enhancements] Compaction Complex Types (STRUCT AND ARRAY)
Cherry-Picked from PR #2516
This closes #2976
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e8d25bb3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e8d25bb3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e8d25bb3
Branch: refs/heads/branch-1.5
Commit: e8d25bb3b80a3deda13bc12773e8565c33747131
Parents: 21762c5
Author: sounakr <so...@gmail.com>
Authored: Tue Jul 17 10:35:32 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:33 2018 +0530
----------------------------------------------------------------------
.../core/datastore/row/WriteStepRowUtil.java | 23 +-
.../core/scan/wrappers/ByteArrayWrapper.java | 10 +
.../complexType/TestCompactionComplexType.scala | 992 +++++++++++++++++++
.../complexType/TestComplexDataType.scala | 32 -
.../CarbonAlterTableCompactionCommand.scala | 7 -
.../processing/datatypes/PrimitiveDataType.java | 10 +-
.../converter/impl/FieldEncoderFactory.java | 2 +-
.../processing/merger/CarbonCompactionUtil.java | 33 +
.../store/CarbonFactDataHandlerModel.java | 89 +-
.../util/CarbonDataProcessorUtil.java | 7 +-
10 files changed, 1142 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java
index fe4e10e..49716ac 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java
@@ -59,29 +59,40 @@ public class WriteStepRowUtil {
}
converted[DICTIONARY_DIMENSION] = dictDimensions;
+ Object[] noDictAndComplexKeys =
+ new Object[segmentProperties.getNumberOfNoDictionaryDimension() + segmentProperties
+ .getComplexDimensions().size()];
+
byte[][] noDictionaryKeys = ((ByteArrayWrapper) row[0]).getNoDictionaryKeys();
- Object[] noDictKeys = new Object[noDictionaryKeys.length];
for (int i = 0; i < noDictionaryKeys.length; i++) {
// in case of compaction rows are collected from result collector and are in byte[].
// Convert the no dictionary columns to original data,
// as load expects the no dictionary column with original data.
if (DataTypeUtil.isPrimitiveColumn(noDicAndComplexColumns[i].getDataType())) {
- noDictKeys[i] = DataTypeUtil
+ noDictAndComplexKeys[i] = DataTypeUtil
.getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeys[i],
noDicAndComplexColumns[i].getDataType());
// for timestamp the above method will give the original data, so it should be
// converted again to the format to be loaded (without micros)
- if (null != noDictKeys[i]
+ if (null != noDictAndComplexKeys[i]
&& noDicAndComplexColumns[i].getDataType() == DataTypes.TIMESTAMP) {
- noDictKeys[i] = (long) noDictKeys[i] / 1000L;
+ noDictAndComplexKeys[i] = (long) noDictAndComplexKeys[i] / 1000L;
}
} else {
- noDictKeys[i] = noDictionaryKeys[i];
+ noDictAndComplexKeys[i] = noDictionaryKeys[i];
}
}
+ // For Complex Type Columns
+ byte[][] complexKeys = ((ByteArrayWrapper) row[0]).getComplexTypesKeys();
+ for (int i = segmentProperties.getNumberOfNoDictionaryDimension();
+ i < segmentProperties.getNumberOfNoDictionaryDimension() + segmentProperties
+ .getComplexDimensions().size(); i++) {
+ noDictAndComplexKeys[i] = complexKeys[i];
+ }
+
// no dictionary and complex dimension
- converted[NO_DICTIONARY_AND_COMPLEX] = noDictKeys;
+ converted[NO_DICTIONARY_AND_COMPLEX] = noDictAndComplexKeys;
// measure
int measureCount = row.length - 1;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
index 4cce8af..65f29d4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
@@ -90,6 +90,16 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper>, Serializa
return this.noDictionaryKeys;
}
+
+ /**
+ * to get the complex column data
+ *
+ * @return no complex values
+ */
+ public byte[][] getComplexTypesKeys() {
+ return this.complexTypesKeys;
+ }
+
/**
* to generate the hash code
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala
new file mode 100644
index 0000000..6ff8a15
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala
@@ -0,0 +1,992 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.integration.spark.testsuite.complexType
+
+import java.sql.Timestamp
+
+import scala.collection.mutable
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestCompactionComplexType extends QueryTest {
+
+ test("test INT with struct and array, Encoding INT-->BYTE") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
+ Row(2, Row(600, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
+ Row(3, Row(600, "abc", mutable.WrappedArray.make(Array(20, 30, 40))))))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'500$abc$20:30:40')")
+ sql("insert into adaptive values(2,'600$abc$30:30:40')")
+ sql("insert into adaptive values(3,'700$abc$40:30:40')")
+ sql("insert into adaptive values(4,'800$abc$50:30:40')")
+ sql("alter table adaptive compact 'major'").show(200,false)
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
+ Row(2, Row(600, "abc", mutable.WrappedArray.make(Array(30, 30, 40)))),
+ Row(3, Row(700, "abc", mutable.WrappedArray.make(Array(40, 30, 40)))),
+ Row(4, Row(800, "abc", mutable.WrappedArray.make(Array(50, 30, 40))))))
+ }
+
+ test("test INT with struct and array, Encoding INT-->SHORT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400))))))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'500$abc$200:300:400')")
+ sql("insert into adaptive values(2,'600$abc$300:300:400')")
+ sql("insert into adaptive values(3,'700$abc$400:300:400')")
+ sql("insert into adaptive values(4,'800$abc$500:300:400')")
+ sql("alter table adaptive compact 'major'").show(200,false)
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(600, "abc", mutable.WrappedArray.make(Array(300, 300, 400)))),
+ Row(3, Row(700, "abc", mutable.WrappedArray.make(Array(400, 300, 400)))),
+ Row(4, Row(800, "abc", mutable.WrappedArray.make(Array(500, 300, 400))))))
+ }
+
+ test("test INT with struct and array, Encoding INT-->SHORT INT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int2.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000))))))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(2,'70000$abc$2000000:4000000:4000000')")
+ sql("insert into adaptive values(3,'100000$abc$2000000:5000000:4000000')")
+ sql("insert into adaptive values(4,'200000$abc$2000000:6000000:4000000')")
+ sql("alter table adaptive compact 'major'").show(200,false)
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 4000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 5000000, 4000000)))),
+ Row(4, Row(200000, "abc", mutable.WrappedArray.make(Array(2000000, 6000000, 4000000))))))
+ }
+
+ test("test INT with struct and array, Encoding INT-->INT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int3.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(7000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000))))))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
+ sql("insert into adaptive values(2,'700000$abc$210:350:52000000')")
+ sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("insert into adaptive values(4,'10000001$abd$250:450:62000000')")
+ sql("alter table adaptive compact 'major'").show(200,false)
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(210, 350, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(4, Row(10000001, "abd", mutable.WrappedArray.make(Array(250, 450, 62000000))))))
+ }
+
+
+ test("test SMALLINT with struct and array SMALLINT --> BYTE") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:smallint,name:string," +
+ "marks:array<smallint>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'100$abc$20:30:40')")
+ sql("insert into adaptive values(2,'200$abc$30:40:50')")
+ sql("insert into adaptive values(3,'300$abd$30:41:55')")
+ sql("insert into adaptive values(4,'400$abe$30:42:56')")
+ sql("alter table adaptive compact 'major'").show(200,false)
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(100, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
+ Row(2, Row(200, "abc", mutable.WrappedArray.make(Array(30, 40, 50)))),
+ Row(3, Row(300, "abd", mutable.WrappedArray.make(Array(30, 41, 55)))),
+ Row(4, Row(400, "abe", mutable.WrappedArray.make(Array(30, 42, 56))))))
+ }
+
+ test("test SMALLINT with struct and array SMALLINT --> SHORT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:smallint,name:string," +
+ "marks:array<smallint>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'500$abc$200:300:400')")
+ sql("insert into adaptive values(2,'8000$abc$300:410:500')")
+ sql("insert into adaptive values(3,'9000$abee$310:420:400')")
+ sql("insert into adaptive values(4,'9900$abfffffffffffffff$320:430:500')")
+ sql("alter table adaptive compact 'major'").show(200,false)
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 410, 500)))),
+ Row(3, Row(9000, "abee", mutable.WrappedArray.make(Array(310, 420, 400)))),
+ Row(4, Row(9900, "abfffffffffffffff", mutable.WrappedArray.make(Array(320, 430, 500))))))
+ sql("insert into adaptive values(5,'500$abc$200:310:400')")
+ sql("insert into adaptive values(6,'8000$abc$300:310:500')")
+ sql("insert into adaptive values(7,'9000$abee$310:320:400')")
+ sql("insert into adaptive values(8,'9900$abfffffffffffffffeeee$320:330:500')")
+ sql("alter table adaptive compact 'major'").show(200,false)
+ sql("SHOW SEGMENTS FOR TABLE adaptive").show(200,false)
+ sql("clean files for table adaptive").show(200,false)
+ sql("SHOW SEGMENTS FOR TABLE adaptive").show(200,false)
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 410, 500)))),
+ Row(3, Row(9000, "abee", mutable.WrappedArray.make(Array(310, 420, 400)))),
+ Row(4, Row(9900, "abfffffffffffffff", mutable.WrappedArray.make(Array(320, 430, 500)))),
+ Row(5, Row(500, "abc", mutable.WrappedArray.make(Array(200, 310, 400)))),
+ Row(6, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 310, 500)))),
+ Row(7, Row(9000, "abee", mutable.WrappedArray.make(Array(310, 320, 400)))),
+ Row(8, Row(9900, "abfffffffffffffffeeee", mutable.WrappedArray.make(Array(320, 330, 500))))))
+ }
+
+ test("test BigInt with struct and array BIGINT --> BYTE") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:bigint,name:string," +
+ "marks:array<bigint>>) stored by 'carbondata'")
+ sql("insert into adaptive values(11,'1$abc$21:30:40')")
+ sql("insert into adaptive values(12,'1$ab1$22:30:40')")
+ sql("insert into adaptive values(13,'1$ab2$23:30:40')")
+ sql("insert into adaptive values(14,'1$ab3$24:30:40')")
+ sql("insert into adaptive values(15,'1$ab4$25:30:40')")
+ sql("insert into adaptive values(16,'1$ab5$26:30:40')")
+ sql("insert into adaptive values(17,'1$ab6$27:30:40')")
+ sql("insert into adaptive values(18,'1$ab7$28:30:40')")
+ sql("insert into adaptive values(19,'1$ab8$29:30:40')")
+ sql("insert into adaptive values(20,'1$ab9$30:30:40')")
+ sql("insert into adaptive values(21,'1$ab10$31:30:40')")
+ sql("insert into adaptive values(22,'1$ab11$32:30:40')")
+ sql("alter table adaptive compact 'major'").show(200,false)
+ sql("SHOW SEGMENTS FOR TABLE adaptive").show(200,false)
+ sql("clean files for table adaptive").show(200,false)
+ sql("SHOW SEGMENTS FOR TABLE adaptive").show(200,false)
+
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(11, Row(1, "abc", mutable.WrappedArray.make(Array(21, 30, 40)))),
+ Row(12, Row(1, "ab1", mutable.WrappedArray.make(Array(22, 30, 40)))),
+ Row(13, Row(1, "ab2", mutable.WrappedArray.make(Array(23, 30, 40)))),
+ Row(14, Row(1, "ab3", mutable.WrappedArray.make(Array(24, 30, 40)))),
+ Row(15, Row(1, "ab4", mutable.WrappedArray.make(Array(25, 30, 40)))),
+ Row(16, Row(1, "ab5", mutable.WrappedArray.make(Array(26, 30, 40)))),
+ Row(17, Row(1, "ab6", mutable.WrappedArray.make(Array(27, 30, 40)))),
+ Row(18, Row(1, "ab7", mutable.WrappedArray.make(Array(28, 30, 40)))),
+ Row(19, Row(1, "ab8", mutable.WrappedArray.make(Array(29, 30, 40)))),
+ Row(20, Row(1, "ab9", mutable.WrappedArray.make(Array(30, 30, 40)))),
+ Row(21, Row(1, "ab10", mutable.WrappedArray.make(Array(31, 30, 40)))),
+ Row(22, Row(1, "ab11", mutable.WrappedArray.make(Array(32, 30, 40))))
+ ))
+ }
+
+ test("test BigInt with struct and array BIGINT --> SHORT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:bigint,name:string," +
+ "marks:array<bigint>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'500$abc$200:300:400')")
+ sql("insert into adaptive values(2,'8000$abc$300:400:500')")
+ sql("insert into adaptive values(3,'9000$abc$300:400:500')")
+ sql("insert into adaptive values(4,'10000$abc$300:400:500')")
+ sql("alter table adaptive compact'major'")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 400, 500)))),
+ Row(3, Row(9000, "abc", mutable.WrappedArray.make(Array(300, 400, 500)))),
+ Row(4, Row(10000, "abc", mutable.WrappedArray.make(Array(300, 400, 500))))))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:BIGINT,name:string,marks:array<BIGINT>>)" +
+ " " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql("alter table adaptive compact'major'")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400))))
+ ))
+ }
+
+ test("test BigInt with struct and array BIGINT --> SHORT INT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:bigint,name:string," +
+ "marks:array<bigint>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ sql("alter table adaptive compact'major'")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000))))
+ ))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:BIGINT,name:string,marks:array<BIGINT>>)" +
+ " " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int2.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int2.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int2.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int2.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql("alter table adaptive compact'major'")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000))))
+ ))
+ }
+
+ test("test BIGINT with struct and array, Encoding INT-->INT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:BIGINT,name:string,marks:array<BIGINT>>)" +
+ " " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int3.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int3.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int3.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int3.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql("alter table adaptive compact'major'")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(7000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(7000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(7000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(7000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000))))
+ ))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:BIGINT,name:string,marks:array<BIGINT>>)" +
+ " " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
+ sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
+ sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
+ sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
+ sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
+ sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
+ sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
+ sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
+ sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000))))
+ ))
+ }
+
+ test("test Double with Struct and Array DOUBLE --> BYTE") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:double,name:string," +
+ "marks:array<double>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'1.323$abc$2.2:3.3:4.4')")
+ sql("insert into adaptive values(2,'1.324$abc$2.2:3.3:4.4')")
+ sql("insert into adaptive values(3,'1.325$abc$2.2:3.3:4.4')")
+ sql("insert into adaptive values(4,'1.326$abc$2.2:3.3:4.4')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(2, Row(1.324, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(3, Row(1.325, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(4, Row(1.326, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4))))))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:double,name:string,marks:array<double>>)" +
+ " " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double1.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double1.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double1.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double1.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(2, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(3, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(2, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(3, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(2, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(3, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(2, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
+ Row(3, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4))))
+ ))
+ }
+
+ test("test Double with Struct and Array DOUBLE --> SHORT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:double,name:string," +
+ "marks:array<double>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'1.323$abc$20.2:30.3:40.4')")
+ sql("insert into adaptive values(2,'1.324$abc$20.2:30.3:40.5')")
+ sql("insert into adaptive values(3,'1.325$abc$20.2:30.3:40.6')")
+ sql("insert into adaptive values(4,'1.326$abc$20.2:30.3:40.7')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(2, Row(1.324, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.5)))),
+ Row(3, Row(1.325, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.6)))),
+ Row(4, Row(1.326, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.7))))
+ ))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:double,name:string,marks:array<double>>)" +
+ " " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double2.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double2.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double2.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double2.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(3, Row(4.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(3, Row(4.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(3, Row(4.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
+ Row(3, Row(4.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4))))
+ ))
+ }
+
+ test("test Double with Struct and Array DOUBLE --> SHORT INT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:double,name:string," +
+ "marks:array<double>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'10.323$abc$20.2:30.3:501.423')")
+ sql("insert into adaptive values(2,'10.323$abc$20.2:30.3:502.421')")
+ sql("insert into adaptive values(3,'10.323$abc$20.2:30.3:503.422')")
+ sql("insert into adaptive values(4,'10.323$abc$20.2:30.3:504.424')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(10.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 501.423)))),
+ Row(2, Row(10.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 502.421)))),
+ Row(3, Row(10.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 503.422)))),
+ Row(4, Row(10.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 504.424)))) ))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:double,name:string,marks:array<double>>)" +
+ " " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double3.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double3.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double3.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double3.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(3, Row(50.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(3, Row(50.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(3, Row(50.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423)))),
+ Row(3, Row(50.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423))))
+ ))
+ }
+
+ test("test Double with Struct and Array DOUBLE --> INT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:double,name:string," +
+ "marks:array<double>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'1000.323$abc$20.2:30.3:60000.423')")
+ sql("insert into adaptive values(2,'1000.324$abc$20.2:30.3:70000.424')")
+ sql("insert into adaptive values(3,'1000.325$abc$20.2:30.3:80000.425')")
+ sql("insert into adaptive values(4,'1000.326$abc$20.2:30.3:90000.426')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(1000.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 60000.423)))),
+ Row(2, Row(1000.324, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 70000.424)))),
+ Row(3, Row(1000.325, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 80000.425)))),
+ Row(4, Row(1000.326, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 90000.426))))
+ ))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:double,name:string,marks:array<double>>)" +
+ " " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double4.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double4.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double4.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_double4.csv' into table adaptive options('delimiter'='," +
+ "'," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(3, Row(50000.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(3, Row(50000.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(3, Row(50000.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(2, Row(2.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423)))),
+ Row(3, Row(50000.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423))))
+ ))
+ }
+
+ test("test Double with Struct and Array DOUBLE --> DOUBLE") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:double,name:string," +
+ "marks:array<double>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'1.797693134862315$abc$2.2:30.3:1.797693134862315')")
+ sql("insert into adaptive values(2,'1.797693134862316$abc$2.2:30.3:1.797693134862316')")
+ sql("insert into adaptive values(3,'1.797693134862317$abc$2.2:30.3:1.797693134862317')")
+ sql("insert into adaptive values(4,'1.797693134862318$abc$2.2:30.3:1.797693134862318')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1,
+ Row(1.797693134862315,
+ "abc",
+ mutable.WrappedArray.make(Array(2.2, 30.3, 1.797693134862315)))),
+ Row(2,
+ Row(1.797693134862316,
+ "abc",
+ mutable.WrappedArray.make(Array(2.2, 30.3, 1.797693134862316)))),
+ Row(3,
+ Row(1.797693134862317,
+ "abc",
+ mutable.WrappedArray.make(Array(2.2, 30.3, 1.797693134862317)))),
+ Row(4,
+ Row(1.797693134862318,
+ "abc",
+ mutable.WrappedArray.make(Array(2.2, 30.3, 1.797693134862318))))
+ ))
+
+ }
+
+ test("test Decimal with Struct") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:decimal(3,2),name:string>) stored by " +
+ "'carbondata'")
+ sql("insert into adaptive values(1,'3.2$abc')")
+ sql("select * from adaptive").show(false)
+ }
+
+ test("test Decimal with Array") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<name:string," +
+ "marks:array<decimal>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'abc$20.2:30.3:40.4')")
+ sql("select * from adaptive").show(false)
+ }
+
+ test("test Timestamp with Struct") {
+ sql("Drop table if exists adaptive")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+ sql(
+ "create table adaptive(roll int, student struct<id:timestamp,name:string>) stored by " +
+ "'carbondata'")
+ sql("insert into adaptive values(1,'2017/01/01 00:00:00$abc')")
+ sql("insert into adaptive values(2,'2017/01/02 00:00:00$abc')")
+ sql("insert into adaptive values(3,'2017/01/03 00:00:00$abc')")
+ sql("insert into adaptive values(4,'2017/01/04 00:00:00$abc')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(Timestamp.valueOf("2017-01-01 00:00:00.0"), "abc")),
+ Row(2, Row(Timestamp.valueOf("2017-01-02 00:00:00.0"), "abc")),
+ Row(3, Row(Timestamp.valueOf("2017-01-03 00:00:00.0"), "abc")),
+ Row(4, Row(Timestamp.valueOf("2017-01-04 00:00:00.0"), "abc"))
+ ))
+ }
+
+ test("test Timestamp with Array") {
+ sql("Drop table if exists adaptive")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+ sql(
+ "create table adaptive(roll int, student struct<name:string," +
+ "marks:array<timestamp>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'abc1$2017/01/01:2018/01/01')")
+ sql("insert into adaptive values(2,'abc2$2017/01/02:2018/01/03')")
+ sql("insert into adaptive values(3,'abc3$2017/01/04:2018/01/05')")
+ sql("insert into adaptive values(4,'abc4$2017/01/06:2018/01/07')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1,
+ Row("abc1",
+ mutable.WrappedArray
+ .make(Array(Timestamp.valueOf("2017-01-01 00:00:00.0"),
+ Timestamp.valueOf("2018-01-01 00:00:00.0"))))),
+ Row(2,
+ Row("abc2",
+ mutable.WrappedArray
+ .make(Array(Timestamp.valueOf("2017-01-02 00:00:00.0"),
+ Timestamp.valueOf("2018-01-03 00:00:00.0"))))),
+ Row(3,
+ Row("abc3",
+ mutable.WrappedArray
+ .make(Array(Timestamp.valueOf("2017-01-04 00:00:00.0"),
+ Timestamp.valueOf("2018-01-05 00:00:00.0"))))),
+ Row(4,
+ Row("abc4",
+ mutable.WrappedArray
+ .make(Array(Timestamp.valueOf("2017-01-06 00:00:00.0"),
+ Timestamp.valueOf("2018-01-07 00:00:00.0")))))
+ ))
+ }
+
+ test("test DATE with Array") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<name:string," +
+ "marks:array<date>>) stored by 'carbondata'")
+ sql("insert into adaptive values(1,'abc$2017-01-01')")
+ sql("select * from adaptive").show(false)
+ }
+
+ test("test LONG with Array and Struct Encoding LONG --> BYTE") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'11111$abc$20:30:40')")
+ sql("insert into adaptive values(2,'11111$abc$55:65:75')")
+ sql("insert into adaptive values(3,'11111$abc$88:98:8')")
+ sql("insert into adaptive values(4,'11111$abc$99:9:19')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
+ Row(2, Row(11111, "abc", mutable.WrappedArray.make(Array(55, 65, 75)))),
+ Row(3, Row(11111, "abc", mutable.WrappedArray.make(Array(88, 98, 8)))),
+ Row(4, Row(11111, "abc", mutable.WrappedArray.make(Array(99, 9, 19))))
+ ))
+ }
+
+ test("test LONG with Array and Struct Encoding LONG --> SHORT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'11111$abc$200:300:400')")
+ sql("insert into adaptive values(2,'11111$abc$201:301:401')")
+ sql("insert into adaptive values(3,'11111$abc$202:302:402')")
+ sql("insert into adaptive values(4,'11111$abc$203:303:403')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(11111, "abc", mutable.WrappedArray.make(Array(201, 301, 401)))),
+ Row(3, Row(11111, "abc", mutable.WrappedArray.make(Array(202, 302, 402)))),
+ Row(4, Row(11111, "abc", mutable.WrappedArray.make(Array(203, 303, 403))))
+ ))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400))))
+ ))
+ }
+
+ test("test LONG with struct and array, Encoding LONG-->SHORT INT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int2.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000))))))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
+ Row(3, Row(100000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000))))))
+ }
+
+ test("test LONG with struct and array, Encoding LONG-->INT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int3.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(7000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000))))))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
+ sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
+ sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
+ Row(3, Row(10000000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000))))))
+ }
+
+ test("test LONG with struct and array, Encoding LONG-->LONG") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'500000$abc$200:300:52000000000')")
+ sql("insert into adaptive values(2,'700000$abc$200:300:52000000000')")
+ sql("insert into adaptive values(3,'10000000$abc$200:300:52000000000')")
+ sql("select * from adaptive").show(false)
+ }
+
+ test("test SHORT with Array and Struct Encoding SHORT -->BYTE") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:short,name:string,marks:array<short>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'11$abc$20:30:40')")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(11, "abc", mutable.WrappedArray.make(Array(20, 30, 40))))))
+ }
+
+ test("test SHORT with Array and Struct Encoding SHORT --> SHORT") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:SHORT,name:string,marks:array<SHORT>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'11111$abc$200:300:400')")
+ sql("insert into adaptive values(1,'11111$abc$200:300:401')")
+ sql("insert into adaptive values(1,'11111$abc$200:300:402')")
+ sql("insert into adaptive values(1,'11111$abc$200:300:403')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 401)))),
+ Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 402)))),
+ Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 403))))
+ ))
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:SHORT,name:string,marks:array<SHORT>>) " +
+ "stored by 'carbondata'")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql(
+ s"load data inpath '$resourcesPath/adap_int1.csv' into table adaptive options('delimiter'=','," +
+ "'quotechar'='\"','fileheader'='roll,student','complex_delimiter_level_1'='$'," +
+ "'complex_delimiter_level_2'=':')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
+ Row(3, Row(800, "abc", mutable.WrappedArray.make(Array(200, 300, 400))))
+ ))
+ }
+
+ test("test Boolean with Struct and Array") {
+ sql("Drop table if exists adaptive")
+ sql(
+ "create table adaptive(roll int, student struct<id:boolean,name:string," +
+ "marks:array<boolean>>) " +
+ "stored by 'carbondata'")
+ sql("insert into adaptive values(1,'true$abc$false:true:false')")
+ sql("insert into adaptive values(1,'true$abc$false:true:true')")
+ sql("insert into adaptive values(1,'true$abc$false:true:true')")
+ sql("insert into adaptive values(1,'true$abc$false:true:false')")
+ sql("alter table adaptive compact 'major' ")
+ checkAnswer(sql("select * from adaptive"),
+ Seq(Row(1, Row(true, "abc", mutable.WrappedArray.make(Array(false, true, false)))),
+ Row(1, Row(true, "abc", mutable.WrappedArray.make(Array(false, true, true)))),
+ Row(1, Row(true, "abc", mutable.WrappedArray.make(Array(false, true, true)))),
+ Row(1, Row(true, "abc", mutable.WrappedArray.make(Array(false, true, false))))
+ ))
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
index 220451b..f2e33f3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
@@ -858,38 +858,6 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
arrayException.getMessage)
}
- test("test block compaction") {
- sql("DROP TABLE IF EXISTS table1")
- sql(
- "create table table1 (roll int,person Struct<detail:int,age:string,height:double>) stored " +
- "by 'carbondata'")
- sql(
- "load data inpath '" + resourcesPath +
- "/Struct.csv' into table table1 options('delimiter'=','," +
- "'quotechar'='\"','fileheader'='roll,person','complex_delimiter_level_1'='$'," +
- "'complex_delimiter_level_2'='&')")
- sql(
- "load data inpath '" + resourcesPath +
- "/Struct.csv' into table table1 options('delimiter'=','," +
- "'quotechar'='\"','fileheader'='roll,person','complex_delimiter_level_1'='$'," +
- "'complex_delimiter_level_2'='&')")
- val exception = intercept[UnsupportedOperationException](
- sql("alter table table1 compact 'major'"))
- assertResult(
- "Compaction is unsupported for Table containing Complex Columns")(
- exception.getMessage)
- val exception1 = intercept[UnsupportedOperationException](
- sql("alter table table1 compact 'minor'"))
- assertResult(
- "Compaction is unsupported for Table containing Complex Columns")(
- exception1.getMessage)
- val exception2 = intercept[UnsupportedOperationException](
- sql("alter table table1 compact 'custom' where segment.id in (0,1)"))
- assertResult(
- "Compaction is unsupported for Table containing Complex Columns")(
- exception2.getMessage)
- }
-
test("test complex datatype double for encoding") {
sql("DROP TABLE IF EXISTS table1")
sql(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index 1b1d708..f98c0cf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -87,13 +87,6 @@ case class CarbonAlterTableCompactionCommand(
if (!table.getTableInfo.isTransactionalTable) {
throw new MalformedCarbonCommandException("Unsupported operation on non transactional table")
}
-
- if (table.getTableInfo.getFactTable.getListOfColumns.asScala
- .exists(m => m.getDataType.isComplexType)) {
- throw new UnsupportedOperationException(
- "Compaction is unsupported for Table containing Complex Columns")
- }
-
if (CarbonUtil.hasAggregationDataMap(table) ||
(table.isChildDataMap && null == operationContext.getProperty(table.getTableName))) {
// If the compaction request is of 'streaming' type then we need to generate loadCommands
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
index c12ed1c..54c8a91 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
@@ -111,8 +111,6 @@ public class PrimitiveDataType implements GenericDataType<Object> {
private boolean isDictionary;
- private boolean isEmptyBadRecord;
-
private String nullformat;
private boolean isDirectDictionary;
@@ -133,13 +131,12 @@ public class PrimitiveDataType implements GenericDataType<Object> {
* @param isDictionary
*/
public PrimitiveDataType(String name, DataType dataType, String parentName, String columnId,
- boolean isDictionary, String nullFormat, boolean isEmptyBadRecord) {
+ boolean isDictionary, String nullFormat) {
this.name = name;
this.parentname = parentName;
this.columnId = columnId;
this.isDictionary = isDictionary;
this.nullformat = nullFormat;
- this.isEmptyBadRecord = isEmptyBadRecord;
this.dataType = dataType;
}
@@ -154,19 +151,17 @@ public class PrimitiveDataType implements GenericDataType<Object> {
* @param useOnePass
* @param localCache
* @param nullFormat
- * @param isEmptyBadRecords
*/
public PrimitiveDataType(CarbonColumn carbonColumn, String parentName, String columnId,
CarbonDimension carbonDimension, AbsoluteTableIdentifier absoluteTableIdentifier,
DictionaryClient client, Boolean useOnePass, Map<Object, Integer> localCache,
- String nullFormat, Boolean isEmptyBadRecords) {
+ String nullFormat) {
this.name = carbonColumn.getColName();
this.parentname = parentName;
this.columnId = columnId;
this.carbonDimension = carbonDimension;
this.isDictionary = isDictionaryDimension(carbonDimension);
this.nullformat = nullFormat;
- this.isEmptyBadRecord = isEmptyBadRecords;
this.dataType = carbonColumn.getDataType();
DictionaryColumnUniqueIdentifier identifier =
@@ -566,7 +561,6 @@ public class PrimitiveDataType implements GenericDataType<Object> {
dataType.parentname = this.parentname;
dataType.columnId = this.columnId;
dataType.dictionaryGenerator = this.dictionaryGenerator;
- dataType.isEmptyBadRecord = this.isEmptyBadRecord;
dataType.nullformat = this.nullformat;
dataType.setKeySize(this.keySize);
dataType.setSurrogateIndex(this.index);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
index 435cf24..3b4df75 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
@@ -184,7 +184,7 @@ public class FieldEncoderFactory {
} else {
return new PrimitiveDataType(carbonColumn, parentName, carbonColumn.getColumnId(),
(CarbonDimension) carbonColumn, absoluteTableIdentifier, client, useOnePass,
- localCache, nullFormat, isEmptyBadRecords);
+ localCache, nullFormat);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index 63f464e..c0af1a4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -326,6 +326,11 @@ public class CarbonCompactionUtil {
updatedCardinalityList.add(value);
}
updatedColumnSchemaList.add(dimension.getColumnSchema());
+
+ if (dimension.getNumberOfChild() > 0) {
+ fillColumnSchemaListForComplexDims(dimension.getListOfChildDimensions(),
+ updatedColumnSchemaList, updatedCardinalityList, columnCardinalityMap);
+ }
}
// add measures to the column schema list
List<CarbonMeasure> masterSchemaMeasures =
@@ -338,6 +343,34 @@ public class CarbonCompactionUtil {
}
/**
+ * This method is to get the chile dimensions of the complex dimension and
+ * update the cardinality for all complex dimensions
+ *
+ * @param carbonDimensionsList
+ * @param updatedColumnSchemaList
+ * @param updatedCardinalityList
+ * @param columnCardinalityMap
+ */
+ private static void fillColumnSchemaListForComplexDims(List<CarbonDimension> carbonDimensionsList,
+ List<ColumnSchema> updatedColumnSchemaList, List<Integer> updatedCardinalityList,
+ Map<String, Integer> columnCardinalityMap) {
+ for (CarbonDimension carbonDimension : carbonDimensionsList) {
+ Integer value = columnCardinalityMap.get(carbonDimension.getColumnId());
+ if (null == value) {
+ updatedCardinalityList.add(getDimensionDefaultCardinality(carbonDimension));
+ } else {
+ updatedCardinalityList.add(value);
+ }
+ updatedColumnSchemaList.add(carbonDimension.getColumnSchema());
+ List<CarbonDimension> childDims = carbonDimension.getListOfChildDimensions();
+ if (null != childDims && childDims.size() > 0) {
+ fillColumnSchemaListForComplexDims(childDims, updatedColumnSchemaList,
+ updatedCardinalityList, columnCardinalityMap);
+ }
+ }
+ }
+
+ /**
* This method will return the default cardinality based on dimension type
*
* @param dimension
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index f70e749..b502da2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -44,7 +44,10 @@ import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.datamap.DataMapWriterListener;
+import org.apache.carbondata.processing.datatypes.ArrayDataType;
import org.apache.carbondata.processing.datatypes.GenericDataType;
+import org.apache.carbondata.processing.datatypes.PrimitiveDataType;
+import org.apache.carbondata.processing.datatypes.StructDataType;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants;
@@ -370,10 +373,10 @@ public class CarbonFactDataHandlerModel {
int[] formattedCardinality = CarbonUtil
.getFormattedCardinality(segmentProperties.getDimColumnsCardinality(), wrapperColumnSchema);
carbonFactDataHandlerModel.setColCardinality(formattedCardinality);
- //TO-DO Need to handle complex types here .
- Map<Integer, GenericDataType> complexIndexMap =
- new HashMap<Integer, GenericDataType>(segmentProperties.getComplexDimensions().size());
- carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap);
+
+ carbonFactDataHandlerModel.setComplexIndexMap(
+ convertComplexDimensionToGenericDataType(segmentProperties,
+ loadModel.getSerializationNullFormat()));
DataType[] measureDataTypes = new DataType[segmentProperties.getMeasures().size()];
int i = 0;
for (CarbonMeasure msr : segmentProperties.getMeasures()) {
@@ -408,6 +411,84 @@ public class CarbonFactDataHandlerModel {
}
/**
+ * This routine takes the Complex Dimension and convert into generic DataType.
+ *
+ * @param segmentProperties
+ * @param isNullFormat
+ * @return
+ */
+ private static Map<Integer, GenericDataType> convertComplexDimensionToGenericDataType(
+ SegmentProperties segmentProperties, String isNullFormat) {
+ List<CarbonDimension> complexDimensions = segmentProperties.getComplexDimensions();
+ Map<Integer, GenericDataType> complexIndexMap = new HashMap<>(complexDimensions.size());
+ int dimensionCount = -1;
+ if (segmentProperties.getDimensions().size() == 0) {
+ dimensionCount = 0;
+ } else {
+ dimensionCount = segmentProperties.getDimensions().size() - segmentProperties
+ .getNumberOfNoDictionaryDimension() - segmentProperties.getComplexDimensions().size();
+ }
+ for (CarbonDimension carbonDimension : complexDimensions) {
+ if (carbonDimension.isComplex()) {
+ GenericDataType genericDataType;
+ DataType dataType = carbonDimension.getDataType();
+ if (DataTypes.isArrayType(dataType)) {
+ genericDataType =
+ new ArrayDataType(carbonDimension.getColName(), "", carbonDimension.getColumnId());
+ } else if (DataTypes.isStructType(dataType)) {
+ genericDataType =
+ new StructDataType(carbonDimension.getColName(), "", carbonDimension.getColumnId());
+ } else {
+ // Add Primitive type.
+ throw new RuntimeException("Primitive Type should not be coming in first loop");
+ }
+ if (carbonDimension.getNumberOfChild() > 0) {
+ addChildrenForComplex(carbonDimension.getListOfChildDimensions(), genericDataType,
+ isNullFormat);
+ }
+ genericDataType.setOutputArrayIndex(0);
+ complexIndexMap.put(dimensionCount++, genericDataType);
+ }
+
+ }
+ return complexIndexMap;
+ }
+
+ private static void addChildrenForComplex(List<CarbonDimension> listOfChildDimensions,
+ GenericDataType genericDataType, String isNullFormat) {
+ for (CarbonDimension carbonDimension : listOfChildDimensions) {
+ String parentColName =
+ carbonDimension.getColName().substring(0, carbonDimension.getColName().lastIndexOf("."));
+ DataType dataType = carbonDimension.getDataType();
+ if (DataTypes.isArrayType(dataType)) {
+ GenericDataType arrayGeneric =
+ new ArrayDataType(carbonDimension.getColName(), parentColName,
+ carbonDimension.getColumnId());
+ if (carbonDimension.getNumberOfChild() > 0) {
+ addChildrenForComplex(carbonDimension.getListOfChildDimensions(), arrayGeneric,
+ isNullFormat);
+ }
+ genericDataType.addChildren(arrayGeneric);
+ } else if (DataTypes.isStructType(dataType)) {
+ GenericDataType structGeneric =
+ new StructDataType(carbonDimension.getColName(), parentColName,
+ carbonDimension.getColumnId());
+ if (carbonDimension.getNumberOfChild() > 0) {
+ addChildrenForComplex(carbonDimension.getListOfChildDimensions(), structGeneric,
+ isNullFormat);
+ }
+ genericDataType.addChildren(structGeneric);
+ } else {
+ // Primitive Data Type
+ genericDataType.addChildren(
+ new PrimitiveDataType(carbonDimension.getColumnSchema().getColumnName(),
+ dataType, parentColName, carbonDimension.getColumnId(),
+ carbonDimension.getColumnSchema().hasEncoding(Encoding.DICTIONARY), isNullFormat));
+ }
+ }
+ }
+
+ /**
* This method will get the store location for the given path, segment id and partition id
*
* @return data directory path
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e8d25bb3/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 1d1f451..98b2543 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -321,9 +321,6 @@ public final class CarbonDataProcessorUtil {
String nullFormat =
configuration.getDataLoadProperty(DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT)
.toString();
- boolean isEmptyBadRecord = Boolean.parseBoolean(
- configuration.getDataLoadProperty(DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD)
- .toString());
Map<String, GenericDataType> complexTypesMap = new LinkedHashMap<String, GenericDataType>();
String[] hierarchies = complexTypeString.split(CarbonCommonConstants.SEMICOLON_SPC_CHARACTER);
@@ -347,8 +344,8 @@ public final class CarbonDataProcessorUtil {
} else {
g.addChildren(
new PrimitiveDataType(levelInfo[0], DataTypeUtil.valueOf(levelInfo[1]),
- levelInfo[2], levelInfo[4], levelInfo[3].contains("true"), nullFormat,
- isEmptyBadRecord));
+ levelInfo[2], levelInfo[4], levelInfo[3].contains("true"), nullFormat
+ ));
}
}
}
[09/21] carbondata git commit: [CARBONDATA-3157] Added lazy load and
direct vector fill support to Presto
Posted by ra...@apache.org.
[CARBONDATA-3157] Added lazy load and direct vector fill support to Presto
To improve the scan performance, integrate lazy loading and direct fil vector features to Carbon Presto Integration.
This PR also fixes the query fail in case of multiple table join and filters
This closes #2978
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/93268eaf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/93268eaf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/93268eaf
Branch: refs/heads/branch-1.5
Commit: 93268eaf02c504d8058a4ec24bf27fbdd9605e43
Parents: 26a5850
Author: ravipesala <ra...@gmail.com>
Authored: Wed Dec 5 18:34:13 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:33 2018 +0530
----------------------------------------------------------------------
.../safe/AbstractNonDictionaryVectorFiller.java | 43 ++-
.../adaptive/AdaptiveDeltaFloatingCodec.java | 4 +-
.../adaptive/AdaptiveDeltaIntegralCodec.java | 4 +-
.../adaptive/AdaptiveFloatingCodec.java | 4 +-
.../adaptive/AdaptiveIntegralCodec.java | 4 +-
.../encoding/compress/DirectCompressCodec.java | 4 +-
.../RestructureBasedVectorResultCollector.java | 3 +-
.../scan/result/vector/CarbonColumnVector.java | 2 +-
.../vector/impl/CarbonColumnVectorImpl.java | 24 +-
.../AbstractCarbonColumnarVector.java | 2 +-
.../ColumnarVectorWrapperDirectFactory.java | 22 +-
.../vector/impl/directread/SequentialFill.java | 38 +++
.../presto/CarbonColumnVectorWrapper.java | 2 +-
.../carbondata/presto/CarbonVectorBatch.java | 10 +-
.../carbondata/presto/CarbondataPageSource.java | 1 +
.../presto/CarbondataPageSourceProvider.java | 5 +
.../presto/ColumnarVectorWrapperDirect.java | 310 +++++++++++++++++++
.../PrestoCarbonVectorizedRecordReader.java | 9 +-
.../presto/impl/CarbonTableConfig.java | 10 +
.../presto/impl/CarbonTableReader.java | 2 +-
.../presto/readers/BooleanStreamReader.java | 10 +
.../presto/readers/SliceStreamReader.java | 2 +-
.../vectorreader/ColumnarVectorWrapper.java | 2 +-
.../ColumnarVectorWrapperDirect.java | 2 +-
24 files changed, 482 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/AbstractNonDictionaryVectorFiller.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/AbstractNonDictionaryVectorFiller.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/AbstractNonDictionaryVectorFiller.java
index 23ac4a9..38e28ae 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/AbstractNonDictionaryVectorFiller.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/AbstractNonDictionaryVectorFiller.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.impl.directread.ColumnarVectorWrapperDirectWithInvertedIndex;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.SequentialFill;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -83,20 +84,39 @@ class StringVectorFiller extends AbstractNonDictionaryVectorFiller {
@Override
public void fillVector(byte[] data, CarbonColumnVector vector) {
// start position will be used to store the current data position
+ boolean addSequential = vector instanceof ColumnarVectorWrapperDirectWithInvertedIndex
+ || vector instanceof SequentialFill;
+
int localOffset = 0;
ByteUtil.UnsafeComparer comparator = ByteUtil.UnsafeComparer.INSTANCE;
- for (int i = 0; i < numberOfRows; i++) {
- int length = (((data[localOffset] & 0xFF) << 8) | (data[localOffset + 1] & 0xFF));
- localOffset += 2;
- if (comparator.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, 0,
- CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length, data, localOffset, length)) {
- vector.putNull(i);
- } else {
- vector.putArray(i, localOffset, length);
+ // In case of inverted index and sequential fill, add data to vector sequentially instead of
+ // adding offsets and data separately.
+ if (addSequential) {
+ for (int i = 0; i < numberOfRows; i++) {
+ int length = (((data[localOffset] & 0xFF) << 8) | (data[localOffset + 1] & 0xFF));
+ localOffset += 2;
+ if (comparator.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, 0,
+ CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length, data, localOffset, length)) {
+ vector.putNull(i);
+ } else {
+ vector.putByteArray(i, localOffset, length, data);
+ }
+ localOffset += length;
}
- localOffset += length;
+ } else {
+ for (int i = 0; i < numberOfRows; i++) {
+ int length = (((data[localOffset] & 0xFF) << 8) | (data[localOffset + 1] & 0xFF));
+ localOffset += 2;
+ if (comparator.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, 0,
+ CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length, data, localOffset, length)) {
+ vector.putNull(i);
+ } else {
+ vector.putArray(i, localOffset, length);
+ }
+ localOffset += length;
+ }
+ vector.putAllByteArray(data, 0, actualDataLength);
}
- vector.putAllByteArray(data, 0, actualDataLength);
}
}
@@ -111,7 +131,8 @@ class LongStringVectorFiller extends AbstractNonDictionaryVectorFiller {
@Override public void fillVector(byte[] data, CarbonColumnVector vector) {
// start position will be used to store the current data position
- boolean invertedIndex = vector instanceof ColumnarVectorWrapperDirectWithInvertedIndex;
+ boolean invertedIndex = vector instanceof ColumnarVectorWrapperDirectWithInvertedIndex
+ || vector instanceof SequentialFill;
int localOffset = 0;
ByteUtil.UnsafeComparer comparator = ByteUtil.UnsafeComparer.INSTANCE;
if (invertedIndex) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
index fb53dba..97c3bed 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
@@ -40,6 +40,7 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
import org.apache.carbondata.core.scan.result.vector.impl.directread.ColumnarVectorWrapperDirectFactory;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.SequentialFill;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.Encoding;
@@ -316,7 +317,8 @@ public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec {
}
}
- if (deletedRows == null || deletedRows.isEmpty()) {
+ if ((deletedRows == null || deletedRows.isEmpty())
+ && !(vectorInfo.vector instanceof SequentialFill)) {
for (int i = nullBits.nextSetBit(0); i >= 0; i = nullBits.nextSetBit(i + 1)) {
vector.putNull(i);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
index 9ed21f4..888ce87 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
import org.apache.carbondata.core.scan.result.vector.impl.directread.ColumnarVectorWrapperDirectFactory;
import org.apache.carbondata.core.scan.result.vector.impl.directread.ConvertableVector;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.SequentialFill;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.Encoding;
@@ -318,7 +319,8 @@ public class AdaptiveDeltaIntegralCodec extends AdaptiveCodec {
.getDirectVectorWrapperFactory(vector, vectorInfo.invertedIndex, nullBits, deletedRows,
true, false);
fillVector(pageData, vector, vectorDataType, pageDataType, pageSize, vectorInfo);
- if (deletedRows == null || deletedRows.isEmpty()) {
+ if ((deletedRows == null || deletedRows.isEmpty())
+ && !(vectorInfo.vector instanceof SequentialFill)) {
for (int i = nullBits.nextSetBit(0); i >= 0; i = nullBits.nextSetBit(i + 1)) {
vector.putNull(i);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index b16e57d..eb266d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -39,6 +39,7 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
import org.apache.carbondata.core.scan.result.vector.impl.directread.ColumnarVectorWrapperDirectFactory;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.SequentialFill;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.Encoding;
@@ -302,7 +303,8 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
}
}
- if (deletedRows == null || deletedRows.isEmpty()) {
+ if ((deletedRows == null || deletedRows.isEmpty())
+ && !(vectorInfo.vector instanceof SequentialFill)) {
for (int i = nullBits.nextSetBit(0); i >= 0; i = nullBits.nextSetBit(i + 1)) {
vector.putNull(i);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
index 66639fe..37ff7c1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
@@ -42,6 +42,7 @@ import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
import org.apache.carbondata.core.scan.result.vector.impl.directread.ColumnarVectorWrapperDirectFactory;
import org.apache.carbondata.core.scan.result.vector.impl.directread.ConvertableVector;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.SequentialFill;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.Encoding;
@@ -291,7 +292,8 @@ public class AdaptiveIntegralCodec extends AdaptiveCodec {
.getDirectVectorWrapperFactory(vector, vectorInfo.invertedIndex, nullBits, deletedRows,
true, false);
fillVector(pageData, vector, vectorDataType, pageDataType, pageSize, vectorInfo, nullBits);
- if (deletedRows == null || deletedRows.isEmpty()) {
+ if ((deletedRows == null || deletedRows.isEmpty())
+ && !(vectorInfo.vector instanceof SequentialFill)) {
for (int i = nullBits.nextSetBit(0); i >= 0; i = nullBits.nextSetBit(i + 1)) {
vector.putNull(i);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
index 60344d6..a33899f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
import org.apache.carbondata.core.scan.result.vector.impl.directread.ColumnarVectorWrapperDirectFactory;
import org.apache.carbondata.core.scan.result.vector.impl.directread.ConvertableVector;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.SequentialFill;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.format.Encoding;
@@ -239,7 +240,8 @@ public class DirectCompressCodec implements ColumnPageCodec {
.getDirectVectorWrapperFactory(vector, vectorInfo.invertedIndex, nullBits, deletedRows,
true, false);
fillVector(pageData, vector, vectorDataType, pageDataType, pageSize, vectorInfo, nullBits);
- if (deletedRows == null || deletedRows.isEmpty()) {
+ if ((deletedRows == null || deletedRows.isEmpty())
+ && !(vectorInfo.vector instanceof SequentialFill)) {
for (int i = nullBits.nextSetBit(0); i >= 0; i = nullBits.nextSetBit(i + 1)) {
vector.putNull(i);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
index b95bffe..a38b9e8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
@@ -204,7 +204,8 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
} else if (dataType == DataTypes.LONG || dataType == DataTypes.TIMESTAMP) {
vector.putLongs(columnVectorInfo.vectorOffset, columnVectorInfo.size, (long) defaultValue);
} else {
- vector.putBytes(columnVectorInfo.vectorOffset, columnVectorInfo.size, (byte[])defaultValue);
+ vector.putByteArray(columnVectorInfo.vectorOffset, columnVectorInfo.size,
+ (byte[]) defaultValue);
}
} else {
vector.putNulls(columnVectorInfo.vectorOffset, columnVectorInfo.size);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
index 25a2eb1..70a2105 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
@@ -68,7 +68,7 @@ public interface CarbonColumnVector {
void putByte(int rowId, byte value);
- void putBytes(int rowId, int count, byte[] value);
+ void putByteArray(int rowId, int count, byte[] value);
void putBytes(int rowId, int count, byte[] src, int srcIndex);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonColumnVectorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonColumnVectorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonColumnVectorImpl.java
index 98536f6..30d2317 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonColumnVectorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonColumnVectorImpl.java
@@ -47,7 +47,7 @@ public class CarbonColumnVectorImpl implements CarbonColumnVector {
private short[] shorts;
- private BitSet nullBytes;
+ protected BitSet nullBytes;
private DataType dataType;
@@ -69,6 +69,10 @@ public class CarbonColumnVectorImpl implements CarbonColumnVector {
private CarbonColumnVector dictionaryVector;
+ private LazyPageLoader lazyPage;
+
+ private boolean loaded;
+
public CarbonColumnVectorImpl(int batchSize, DataType dataType) {
this.batchSize = batchSize;
nullBytes = new BitSet(batchSize);
@@ -163,7 +167,7 @@ public class CarbonColumnVectorImpl implements CarbonColumnVector {
byteArr[rowId] = value;
}
- @Override public void putBytes(int rowId, int count, byte[] value) {
+ @Override public void putByteArray(int rowId, int count, byte[] value) {
for (int i = 0; i < count; ++i) {
bytes[i + rowId] = value;
}
@@ -208,6 +212,9 @@ public class CarbonColumnVectorImpl implements CarbonColumnVector {
}
@Override public Object getData(int rowId) {
+ if (!loaded) {
+ loadPage();
+ }
if (nullBytes.get(rowId)) {
return null;
}
@@ -243,6 +250,9 @@ public class CarbonColumnVectorImpl implements CarbonColumnVector {
}
public Object getDataArray() {
+ if (!loaded) {
+ loadPage();
+ }
if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return byteArr;
} else if (dataType == DataTypes.SHORT) {
@@ -291,6 +301,7 @@ public class CarbonColumnVectorImpl implements CarbonColumnVector {
} else {
Arrays.fill(data, null);
}
+ loaded = false;
}
@@ -367,7 +378,14 @@ public class CarbonColumnVectorImpl implements CarbonColumnVector {
}
@Override public void setLazyPage(LazyPageLoader lazyPage) {
- lazyPage.loadPage();
+ this.lazyPage = lazyPage;
+ }
+
+ public void loadPage() {
+ if (lazyPage != null) {
+ lazyPage.loadPage();
+ }
+ loaded = true;
}
@Override public void putArray(int rowId, int offset, int length) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/AbstractCarbonColumnarVector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/AbstractCarbonColumnarVector.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/AbstractCarbonColumnarVector.java
index 4c783b4..9083a20 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/AbstractCarbonColumnarVector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/AbstractCarbonColumnarVector.java
@@ -59,7 +59,7 @@ public abstract class AbstractCarbonColumnarVector
}
@Override
- public void putBytes(int rowId, int count, byte[] value) {
+ public void putByteArray(int rowId, int count, byte[] value) {
throw new UnsupportedOperationException("Not allowed from here " + getClass().getName());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectFactory.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectFactory.java
index 84e42f5..f6d2941 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectFactory.java
@@ -41,15 +41,25 @@ public final class ColumnarVectorWrapperDirectFactory {
public static CarbonColumnVector getDirectVectorWrapperFactory(CarbonColumnVector columnVector,
int[] invertedIndex, BitSet nullBitset, BitSet deletedRows, boolean isnullBitsExists,
boolean isDictVector) {
- if ((invertedIndex != null && invertedIndex.length > 0) && (deletedRows == null || deletedRows
- .isEmpty())) {
+ // If it is sequential data filler then add the null bitset.
+ if (columnVector instanceof SequentialFill) {
+ // If it has inverted index then create a dummy delete rows bitset so that it goes to
+ // ColumnarVectorWrapperDirectWithDeleteDeltaAndInvertedIndex, here it does the sequential
+ // filling using another vector.
+ if ((invertedIndex != null && invertedIndex.length > 0)) {
+ if (deletedRows == null) {
+ deletedRows = new BitSet();
+ }
+ } else if (deletedRows == null) {
+ ((SequentialFill) columnVector).setNullBits(nullBitset);
+ }
+ }
+ if ((invertedIndex != null && invertedIndex.length > 0) && (deletedRows == null)) {
return new ColumnarVectorWrapperDirectWithInvertedIndex(columnVector, invertedIndex,
isnullBitsExists);
- } else if ((invertedIndex == null || invertedIndex.length == 0) && (deletedRows != null
- && !deletedRows.isEmpty())) {
+ } else if ((invertedIndex == null || invertedIndex.length == 0) && deletedRows != null) {
return new ColumnarVectorWrapperDirectWithDeleteDelta(columnVector, deletedRows, nullBitset);
- } else if ((invertedIndex != null && invertedIndex.length > 0) && (deletedRows != null
- && !deletedRows.isEmpty())) {
+ } else if ((invertedIndex != null && invertedIndex.length > 0) && deletedRows != null) {
return new ColumnarVectorWrapperDirectWithDeleteDeltaAndInvertedIndex(columnVector,
deletedRows, invertedIndex, nullBitset, isnullBitsExists, isDictVector);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/SequentialFill.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/SequentialFill.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/SequentialFill.java
new file mode 100644
index 0000000..a0df68c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/SequentialFill.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.result.vector.impl.directread;
+
+import java.util.BitSet;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * It is sort of a marker interface to let execution engine know that it is appendable/sequential
+ * data adding vector. It means we cannot add random rowids to it.
+ */
+@InterfaceStability.Evolving
+@InterfaceAudience.Internal
+public interface SequentialFill {
+
+ /**
+ * Set the null bitset
+ * @param nullBits
+ */
+ void setNullBits(BitSet nullBits);
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
index f001488..b43676c 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
@@ -157,7 +157,7 @@ public class CarbonColumnVectorWrapper implements CarbonColumnVector {
}
}
- @Override public void putBytes(int rowId, int count, byte[] value) {
+ @Override public void putByteArray(int rowId, int count, byte[] value) {
for (int i = 0; i < count; i++) {
if (!filteredRows[rowId]) {
columnVector.putByteArray(counter++, value);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
index 2a0ab75..fb8300a 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
@@ -21,6 +21,7 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.DecimalType;
@@ -68,8 +69,13 @@ public class CarbonVectorBatch {
}
public static CarbonVectorBatch allocate(StructField[] schema,
- CarbonDictionaryDecodeReadSupport readSupport) {
- return new CarbonVectorBatch(schema, readSupport, DEFAULT_BATCH_SIZE);
+ CarbonDictionaryDecodeReadSupport readSupport, boolean isDirectFill) {
+ if (isDirectFill) {
+ return new CarbonVectorBatch(schema, readSupport,
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
+ } else {
+ return new CarbonVectorBatch(schema, readSupport,DEFAULT_BATCH_SIZE);
+ }
}
private CarbonColumnVectorImpl createDirectStreamReader(int batchSize, DataType dataType,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
index 51677de..93de394 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
@@ -161,6 +161,7 @@ class CarbondataPageSource implements ConnectorPageSource {
}
checkState(batchId == expectedBatchId);
try {
+ vectorReader.getColumnarBatch().column(columnIndex).loadPage();
PrestoVectorBlockBuilder blockBuilder =
(PrestoVectorBlockBuilder) vectorReader.getColumnarBatch().column(columnIndex);
blockBuilder.setBatchSize(lazyBlock.getPositionCount());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
index 3ec815d..bef246e 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
@@ -100,6 +100,11 @@ public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider
checkArgument(carbondataSplit.getConnectorId().equals(connectorId),
"split is not for this connector");
QueryModel queryModel = createQueryModel(carbondataSplit, columns);
+ if (carbonTableReader.config.getPushRowFilter() == null ||
+ carbonTableReader.config.getPushRowFilter().equalsIgnoreCase("false")) {
+ queryModel.setDirectVectorFill(true);
+ queryModel.setPreFetchData(false);
+ }
QueryExecutor queryExecutor =
QueryExecutorFactory.getQueryExecutor(queryModel, new Configuration());
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/ColumnarVectorWrapperDirect.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/ColumnarVectorWrapperDirect.java b/integration/presto/src/main/java/org/apache/carbondata/presto/ColumnarVectorWrapperDirect.java
new file mode 100644
index 0000000..9a381cd
--- /dev/null
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/ColumnarVectorWrapperDirect.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.presto;
+
+import java.math.BigDecimal;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
+import org.apache.carbondata.core.scan.result.vector.impl.CarbonColumnVectorImpl;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.SequentialFill;
+import org.apache.carbondata.core.scan.scanner.LazyPageLoader;
+
+/**
+ * Fills the vector directly with out considering any deleted rows.
+ */
+class ColumnarVectorWrapperDirect implements CarbonColumnVector, SequentialFill {
+ /**
+ * It is adapter class of complete ColumnarBatch.
+ */
+ protected CarbonColumnVectorImpl columnVector;
+
+ /**
+ * It is current block file datatype used for alter table scenarios when datatype changes.
+ */
+ private DataType blockDataType;
+
+ private CarbonColumnVector dictionaryVector;
+
+ private BitSet nullBitset;
+
+ ColumnarVectorWrapperDirect(CarbonColumnVectorImpl columnVector) {
+ this.columnVector = columnVector;
+ this.dictionaryVector = columnVector.getDictionaryVector();
+ this.nullBitset = new BitSet();
+ }
+
+ @Override public void setNullBits(BitSet nullBits) {
+ this.nullBitset = nullBits;
+ }
+
+ @Override public void putBoolean(int rowId, boolean value) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putBoolean(rowId, value);
+ }
+ }
+
+ @Override public void putFloat(int rowId, float value) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putFloat(rowId, value);
+ }
+ }
+
+ @Override public void putShort(int rowId, short value) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putShort(rowId, value);
+ }
+ }
+
+ @Override public void putShorts(int rowId, int count, short value) {
+ columnVector.putShorts(rowId, count, value);
+
+ }
+
+ @Override public void putInt(int rowId, int value) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putInt(rowId, value);
+ }
+ }
+
+ @Override public void putInts(int rowId, int count, int value) {
+ columnVector.putInts(rowId, count, value);
+ }
+
+ @Override public void putLong(int rowId, long value) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putLong(rowId, value);
+ }
+ }
+
+ @Override public void putLongs(int rowId, int count, long value) {
+ columnVector.putLongs(rowId, count, value);
+ }
+
+ @Override public void putDecimal(int rowId, BigDecimal value, int precision) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putDecimal(rowId, value, precision);
+ }
+ }
+
+ @Override public void putDecimals(int rowId, int count, BigDecimal value, int precision) {
+ for (int i = 0; i < count; i++) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putDecimal(rowId, value, precision);
+ }
+ rowId++;
+ }
+ }
+
+ @Override public void putDouble(int rowId, double value) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putDouble(rowId, value);
+ }
+ }
+
+ @Override public void putDoubles(int rowId, int count, double value) {
+ columnVector.putDoubles(rowId, count, value);
+ }
+
+ @Override public void putByteArray(int rowId, byte[] value) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putByteArray(rowId, value);
+ }
+ }
+
+ @Override
+ public void putByteArray(int rowId, int count, byte[] value) {
+ for (int i = 0; i < count; i++) {
+ columnVector.putByteArray(rowId++, value);
+ }
+ }
+
+ @Override public void putByteArray(int rowId, int offset, int length, byte[] value) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putByteArray(rowId, offset, length, value);
+ }
+ }
+
+ @Override public void putNull(int rowId) {
+ columnVector.putNull(rowId);
+ }
+
+ @Override public void putNulls(int rowId, int count) {
+ columnVector.putNulls(rowId, count);
+ }
+
+ @Override public void putNotNull(int rowId) {
+ columnVector.putNotNull(rowId);
+ }
+
+ @Override public void putNotNull(int rowId, int count) {
+ }
+
+ @Override public boolean isNull(int rowId) {
+ return columnVector.isNullAt(rowId);
+ }
+
+ @Override public void putObject(int rowId, Object obj) {
+ throw new UnsupportedOperationException(
+ "Not supported this opeartion from " + this.getClass().getName());
+ }
+
+ @Override public Object getData(int rowId) {
+ throw new UnsupportedOperationException(
+ "Not supported this opeartion from " + this.getClass().getName());
+ }
+
+ @Override public void reset() {
+ if (null != dictionaryVector) {
+ dictionaryVector.reset();
+ }
+ }
+
+ @Override public DataType getType() {
+ return columnVector.getType();
+ }
+
+ @Override public DataType getBlockDataType() {
+ return blockDataType;
+ }
+
+ @Override public void setBlockDataType(DataType blockDataType) {
+ this.blockDataType = blockDataType;
+ }
+
+ @Override public void setDictionary(CarbonDictionary dictionary) {
+ columnVector.setDictionary(dictionary);
+ }
+
+ @Override public boolean hasDictionary() {
+ return columnVector.hasDictionary();
+ }
+
+
+ @Override public CarbonColumnVector getDictionaryVector() {
+ return dictionaryVector;
+ }
+
+ @Override public void putByte(int rowId, byte value) {
+ columnVector.putByte(rowId, value);
+ }
+
+ @Override public void setFilteredRowsExist(boolean filteredRowsExist) {
+ // Leave it, as it does not need to do anything here.
+ }
+
+ @Override public void putFloats(int rowId, int count, float[] src, int srcIndex) {
+ for (int i = 0; i < count; i++) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putFloat(rowId, src[i]);
+ }
+ rowId++;
+ }
+ }
+
+ @Override public void putShorts(int rowId, int count, short[] src, int srcIndex) {
+ for (int i = 0; i < count; i++) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putShort(rowId, src[i]);
+ }
+ rowId++;
+ }
+ }
+
+ @Override public void putInts(int rowId, int count, int[] src, int srcIndex) {
+ for (int i = 0; i < count; i++) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putInt(rowId, src[i]);
+ }
+ rowId++;
+ }
+ }
+
+ @Override public void putLongs(int rowId, int count, long[] src, int srcIndex) {
+ for (int i = 0; i < count; i++) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putLong(rowId, src[i]);
+ }
+ rowId++;
+ }
+ }
+
+ @Override public void putDoubles(int rowId, int count, double[] src, int srcIndex) {
+ for (int i = 0; i < count; i++) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putDouble(rowId, src[i]);
+ }
+ rowId++;
+ }
+ }
+
+ @Override public void putBytes(int rowId, int count, byte[] src, int srcIndex) {
+ for (int i = 0; i < count; i++) {
+ if (nullBitset.get(rowId)) {
+ columnVector.putNull(rowId);
+ } else {
+ columnVector.putByte(rowId, src[i]);
+ }
+ rowId++;
+ }
+ }
+
+ @Override public void setLazyPage(LazyPageLoader lazyPage) {
+ columnVector.setLazyPage(lazyPage);
+ }
+
+ @Override public void putArray(int rowId, int offset, int length) {
+ columnVector.putArray(rowId, offset, length);
+ }
+
+ @Override public void putAllByteArray(byte[] data, int offset, int length) {
+ columnVector.putAllByteArray(data, offset, length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
index 4e2d36c..08401df 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
@@ -214,11 +214,16 @@ class PrestoCarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
}
}
- columnarBatch = CarbonVectorBatch.allocate(fields, readSupport);
+ columnarBatch =
+ CarbonVectorBatch.allocate(fields, readSupport, queryModel.isDirectVectorFill());
CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
boolean[] filteredRows = new boolean[columnarBatch.capacity()];
for (int i = 0; i < fields.length; i++) {
- vectors[i] = new CarbonColumnVectorWrapper(columnarBatch.column(i), filteredRows);
+ if (queryModel.isDirectVectorFill()) {
+ vectors[i] = new ColumnarVectorWrapperDirect(columnarBatch.column(i));
+ } else {
+ vectors[i] = new CarbonColumnVectorWrapper(columnarBatch.column(i), filteredRows);
+ }
}
carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
index 4d18184..ab1c871 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java
@@ -43,6 +43,7 @@ public class CarbonTableConfig {
private String s3N_acesssKey;
private String s3N_secretKey;
private String endPoint;
+ private String pushRowFilter;
@NotNull public String getDbPath() {
@@ -195,4 +196,13 @@ public class CarbonTableConfig {
this.endPoint = endPoint;
return this;
}
+
+ public String getPushRowFilter() {
+ return pushRowFilter;
+ }
+
+ @Config("carbon.push.rowfilters.for.vector")
+ public void setPushRowFilter(String pushRowFilter) {
+ this.pushRowFilter = pushRowFilter;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index 7baf7ea..546bf9b 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -412,7 +412,7 @@ public class CarbonTableReader {
List<CarbonLocalMultiBlockSplit> multiBlockSplitList = new ArrayList<>();
CarbonTable carbonTable = tableCacheModel.carbonTable;
TableInfo tableInfo = tableCacheModel.carbonTable.getTableInfo();
- Configuration config = FileFactory.getConfiguration();
+ Configuration config = new Configuration();
config.set(CarbonTableInputFormat.INPUT_SEGMENT_NUMBERS, "");
String carbonTablePath = carbonTable.getAbsoluteTableIdentifier().getTablePath();
config.set(CarbonTableInputFormat.INPUT_DIR, carbonTablePath);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
index b523064..481ab27 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
@@ -64,6 +64,16 @@ public class BooleanStreamReader extends CarbonColumnVectorImpl
}
}
+ @Override public void putByte(int rowId, byte value) {
+ type.writeBoolean(builder, value == 1);
+ }
+
+ @Override public void putBytes(int rowId, int count, byte[] src, int srcIndex) {
+ for (int i = 0; i < count; i++) {
+ type.writeBoolean(builder, src[srcIndex++] == 1);
+ }
+ }
+
@Override public void putBoolean(int rowId, boolean value) {
type.writeBoolean(builder, value);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
index 04e5bb3..d9c7ad3 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
@@ -80,8 +80,8 @@ public class SliceStreamReader extends CarbonColumnVectorImpl implements PrestoV
int[] dictOffsets = new int[dictionary.getDictionarySize() + 1];
int size = 0;
for (int i = 0; i < dictionary.getDictionarySize(); i++) {
+ dictOffsets[i] = size;
if (dictionary.getDictionaryValue(i) != null) {
- dictOffsets[i] = size;
size += dictionary.getDictionaryValue(i).length;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
index 7520ef6..dbad225 100644
--- a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
+++ b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
@@ -177,7 +177,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
}
}
- @Override public void putBytes(int rowId, int count, byte[] value) {
+ @Override public void putByteArray(int rowId, int count, byte[] value) {
for (int i = 0; i < count; i++) {
if (!filteredRows[rowId]) {
sparkColumnVectorProxy.putByteArray(counter++, value);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/93268eaf/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapperDirect.java
----------------------------------------------------------------------
diff --git a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapperDirect.java b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapperDirect.java
index c50d060..413c5f0 100644
--- a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapperDirect.java
+++ b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapperDirect.java
@@ -119,7 +119,7 @@ class ColumnarVectorWrapperDirect implements CarbonColumnVector {
}
@Override
- public void putBytes(int rowId, int count, byte[] value) {
+ public void putByteArray(int rowId, int count, byte[] value) {
for (int i = 0; i < count; i++) {
sparkColumnVectorProxy.putByteArray(rowId, value);
rowId++;
[13/21] carbondata git commit: [CARBONDATA-3158] support
presto-carbon to read sdk cabron files
Posted by ra...@apache.org.
[CARBONDATA-3158] support presto-carbon to read sdk cabron files
problem:
Currently, carbon SDK files output (files without metadata folder and its
contents) are read by spark using an external table with carbon session.
But presto carbon integration doesn't support that. It can currently read
only the transactional table output files.
solution:
Hence we can enhance presto to read SDK output files. This will increase
the use cases for presto-carbon integration.
The above scenario can be achieved by inferring schema if metadata folder
not exists and
setting read committed scope to LatestFilesReadCommittedScope, if
non-transctional table output files are present.
This closes #2982
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/992b8f40
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/992b8f40
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/992b8f40
Branch: refs/heads/branch-1.5
Commit: 992b8f4024a7ecacefd61dd805e29a60b5fd3d66
Parents: e8d25bb
Author: ajantha-bhat <aj...@gmail.com>
Authored: Fri Dec 7 18:37:10 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:34 2018 +0530
----------------------------------------------------------------------
integration/presto/pom.xml | 6 +
.../presto/CarbondataPageSourceProvider.java | 3 +
.../presto/impl/CarbonTableReader.java | 74 +++---
.../PrestoAllDataTypeLocalDictTest.scala | 30 +--
.../integrationtest/PrestoAllDataTypeTest.scala | 102 ++++----
.../PrestoTestNonTransactionalTableFiles.scala | 233 +++++++++++++++++++
.../carbondata/presto/server/PrestoServer.scala | 35 ++-
7 files changed, 378 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/992b8f40/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index d0556b9..1c1b91c 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -487,6 +487,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-store-sdk</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
<version>1.4.0</version>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/992b8f40/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
index bef246e..96024e4 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java
@@ -139,6 +139,9 @@ public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider
Configuration conf = new Configuration();
conf.set(CarbonTableInputFormat.INPUT_SEGMENT_NUMBERS, "");
String carbonTablePath = carbonTable.getAbsoluteTableIdentifier().getTablePath();
+ CarbonTableInputFormat
+ .setTransactionalTable(conf, carbonTable.getTableInfo().isTransactionalTable());
+ CarbonTableInputFormat.setTableInfo(conf, carbonTable.getTableInfo());
conf.set(CarbonTableInputFormat.INPUT_DIR, carbonTablePath);
conf.set("query.id", queryId);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/992b8f40/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index 546bf9b..9677839 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -53,8 +53,10 @@ import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.api.CarbonInputFormat;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
import org.apache.carbondata.presto.PrestoFilterUtil;
@@ -284,18 +286,6 @@ public class CarbonTableReader {
if (carbonFileList == null) {
updateSchemaList();
}
- try {
- if (isKeyExists
- && !FileFactory.isFileExist(
- CarbonTablePath.getSchemaFilePath(
- carbonCache.get().get(schemaTableName).carbonTable.getTablePath()), fileType)) {
- removeTableFromCache(schemaTableName);
- throw new TableNotFoundException(schemaTableName);
- }
- } catch (IOException e) {
- throw new RuntimeException();
- }
-
if (isKeyExists) {
CarbonTableCacheModel carbonTableCacheModel = carbonCache.get().get(schemaTableName);
if (carbonTableCacheModel != null
@@ -345,14 +335,15 @@ public class CarbonTableReader {
* @return the CarbonTable instance which contains all the needed metadata for a table.
*/
private CarbonTable parseCarbonMetadata(SchemaTableName table) {
- CarbonTable result = null;
+ CarbonTable result;
try {
CarbonTableCacheModel cache = carbonCache.get().get(table);
if (cache == null) {
cache = new CarbonTableCacheModel();
}
- if (cache.isValid()) return cache.carbonTable;
-
+ if (cache.isValid()) {
+ return cache.carbonTable;
+ }
// If table is not previously cached, then:
// Step 1: get store path of the table and cache it.
@@ -364,23 +355,38 @@ public class CarbonTableReader {
String tablePath = storePath + "/" + carbonTableIdentifier.getDatabaseName() + "/"
+ carbonTableIdentifier.getTableName();
- //Step 2: read the metadata (tableInfo) of the table.
- ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
- // TBase is used to read and write thrift objects.
- // TableInfo is a kind of TBase used to read and write table information.
- // TableInfo is generated by thrift,
- // see schema.thrift under format/src/main/thrift for details.
- public TBase create() {
- return new org.apache.carbondata.format.TableInfo();
+ String metadataPath = CarbonTablePath.getSchemaFilePath(tablePath);
+ boolean isTransactionalTable = false;
+ try {
+ FileFactory.FileType fileType = FileFactory.getFileType(metadataPath);
+ if (FileFactory.getCarbonFile(metadataPath, fileType).isFileExist(metadataPath, fileType)) {
+ // If metadata folder exists, it is a transactional table
+ isTransactionalTable = true;
}
- };
- ThriftReader thriftReader =
- new ThriftReader(CarbonTablePath.getSchemaFilePath(tablePath), createTBase);
- thriftReader.open();
- org.apache.carbondata.format.TableInfo tableInfo =
- (org.apache.carbondata.format.TableInfo) thriftReader.read();
- thriftReader.close();
-
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ org.apache.carbondata.format.TableInfo tableInfo;
+ if (isTransactionalTable) {
+ //Step 2: read the metadata (tableInfo) of the table.
+ ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
+ // TBase is used to read and write thrift objects.
+ // TableInfo is a kind of TBase used to read and write table information.
+ // TableInfo is generated by thrift,
+ // see schema.thrift under format/src/main/thrift for details.
+ public TBase create() {
+ return new org.apache.carbondata.format.TableInfo();
+ }
+ };
+ ThriftReader thriftReader =
+ new ThriftReader(CarbonTablePath.getSchemaFilePath(tablePath), createTBase);
+ thriftReader.open();
+ tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
+ thriftReader.close();
+ } else {
+ tableInfo = CarbonUtil
+ .inferSchema(tablePath, table.getTableName(), false, FileFactory.getConfiguration());
+ }
// Step 3: convert format level TableInfo to code level TableInfo
SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
// wrapperTableInfo is the code level information of a table in carbondata core,
@@ -389,6 +395,8 @@ public class CarbonTableReader {
.fromExternalToWrapperTableInfo(tableInfo, table.getSchemaName(), table.getTableName(),
tablePath);
+ wrapperTableInfo.setTransactionalTable(isTransactionalTable);
+
// Step 4: Load metadata info into CarbonMetadata
CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
@@ -419,6 +427,8 @@ public class CarbonTableReader {
config.set(CarbonTableInputFormat.DATABASE_NAME, carbonTable.getDatabaseName());
config.set(CarbonTableInputFormat.TABLE_NAME, carbonTable.getTableName());
config.set("query.id", queryId);
+ CarbonInputFormat.setTransactionalTable(config, carbonTable.isTransactionalTable());
+ CarbonInputFormat.setTableInfo(config, carbonTable.getTableInfo());
JobConf jobConf = new JobConf(config);
List<PartitionSpec> filteredPartitions = new ArrayList();
@@ -470,7 +480,7 @@ public class CarbonTableReader {
}
} catch (IOException e) {
- throw new RuntimeException("Error creating Splits from CarbonTableInputFormat", e);
+ throw new RuntimeException(e);
}
return multiBlockSplitList;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/992b8f40/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
index 356c0e8..3f4d839 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
@@ -40,7 +40,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
+ "../../../..").getCanonicalPath
private val storePath = s"$rootPath/integration/presto/target/store"
private val systemPath = s"$rootPath/integration/presto/target/system"
- private val PrestoServer = new PrestoServer
+ private val prestoServer = new PrestoServer
// Table schema:
// +-------------+----------------+-------------+------------+
@@ -80,15 +80,15 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
s"$rootPath/integration/presto/src/test/resources/alldatatype.csv", true)
logger.info(s"\nCarbon store is created at location: $storePath")
cleanUp
- PrestoServer.startServer(storePath)
+ prestoServer.startServer(storePath)
}
override def afterAll(): Unit = {
- PrestoServer.stopServer()
+ prestoServer.stopServer()
}
test("select string type with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE ORDER BY NAME")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "akash"),
Map("NAME" -> "anubhav"),
@@ -106,7 +106,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
}
test("test and filter clause with greater than expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>1234 AND ID>2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -133,7 +133,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
}
test("test and filter clause with greater than equal to expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -174,7 +174,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.toString() equals expectedResult.toString())
}
test("test and filter clause with less than equal to expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS<=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -199,7 +199,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.toString() equals expectedResult.toString())
}
test("test equal to expression on decimal value") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID FROM TESTDB.TESTTABLE WHERE BONUS=1234.444")
@@ -208,7 +208,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult equals expectedResult)
}
test("test less than expression with and operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>1234 AND ID<2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -224,7 +224,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.toString().equals(expectedResult.toString()))
}
test("test the result for in clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME from testdb.testtable WHERE PHONETYPE IN('phone1848','phone706')")
val expectedResult: List[Map[String, Any]] = List(
Map("NAME" -> "geetika"),
@@ -234,7 +234,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.equals(expectedResult))
}
test("test the result for not in clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT NAME from testdb.testtable WHERE PHONETYPE NOT IN('phone1848','phone706')")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "anubhav"),
@@ -248,21 +248,21 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
assert(actualResult.equals(expectedResult))
}
test("test for null operator on date data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT ID FROM TESTDB.TESTTABLE WHERE DATE IS NULL")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 9),Map("ID" -> null))
assert(actualResult.equals(expectedResult))
}
test("test for not null operator on date data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DATE IS NOT NULL AND ID=9")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"))
assert(actualResult.equals(expectedResult))
}
test("test for not null operator on timestamp type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DOB IS NOT NULL AND ID=9")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"),
Map("NAME" -> "jitesh"))
@@ -271,7 +271,7 @@ class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll
}
test("test timestamp datatype using cast operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME AS RESULT FROM TESTDB.TESTTABLE WHERE DOB = CAST('2016-04-14 15:00:09' AS TIMESTAMP)")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "jatin"))
assert(actualResult.equals(expectedResult))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/992b8f40/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
index 97539c0..7bab795 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
@@ -40,7 +40,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
+ "../../../..").getCanonicalPath
private val storePath = s"$rootPath/integration/presto/target/store"
private val systemPath = s"$rootPath/integration/presto/target/system"
- private val PrestoServer = new PrestoServer
+ private val prestoServer = new PrestoServer
// Table schema:
// +-------------+----------------+-------------+------------+
@@ -80,77 +80,77 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
s"$rootPath/integration/presto/src/test/resources/alldatatype.csv")
logger.info(s"\nCarbon store is created at location: $storePath")
cleanUp
- PrestoServer.startServer(storePath)
+ prestoServer.startServer(storePath, "testdb")
}
override def afterAll(): Unit = {
- PrestoServer.stopServer()
+ prestoServer.stopServer()
}
test("test the result for count(*) in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT COUNT(*) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 11))
assert(actualResult.equals(expectedResult))
}
test("test the result for count() clause with distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT COUNT(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 9))
assert(actualResult.equals(expectedResult))
}
test("test the result for sum()in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT SUM(ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 54))
assert(actualResult.equals(expectedResult))
}
test("test the result for sum() wiTh distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT SUM(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 45))
assert(actualResult.equals(expectedResult))
}
test("test the result for avg() with distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT AVG(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 5))
assert(actualResult.equals(expectedResult))
}
test("test the result for min() with distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT MIN(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 1))
assert(actualResult.equals(expectedResult))
}
test("test the result for max() with distinct operator in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT MAX(DISTINCT ID) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 9))
assert(actualResult.equals(expectedResult))
}
test("test the result for count()clause with distinct operator on decimal column in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT COUNT(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10))
assert(actualResult.equals(expectedResult))
}
test("test the result for count()clause with out distinct operator on decimal column in presto")
{
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT COUNT(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10))
assert(actualResult.equals(expectedResult))
}
test("test the result for sum()with out distinct operator for decimal column in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT SUM(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 20774.6475))
assert(actualResult.toString().equals(expectedResult.toString()))
}
test("test the result for sum() with distinct operator for decimal column in presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT SUM(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 20774.6475))
assert(
@@ -158,14 +158,14 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
expectedResult.head("RESULT").toString.toDouble)
}
test("test the result for avg() with distinct operator on decimal on presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT AVG(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 2077.4648))
assert(actualResult.toString.equals(expectedResult.toString))
}
test("test the result for min() with distinct operator in decimalType of presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT MIN(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map(
"RESULT" -> java.math.BigDecimal.valueOf(500.414).setScale(4)))
@@ -173,14 +173,14 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the result for max() with distinct operator in decimalType of presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT MAX(BONUS) AS RESULT FROM TESTDB.TESTTABLE ")
val expectedResult: List[Map[String, Any]] = List(Map(
"RESULT" -> java.math.BigDecimal.valueOf(9999.999).setScale(4)))
assert(actualResult.equals(expectedResult))
}
test("select decimal data type with ORDER BY clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT DISTINCT BONUS FROM TESTDB.TESTTABLE ORDER BY BONUS limit 3 ")
val expectedResult: List[Map[String, Any]] = List(Map(
"BONUS" -> java.math.BigDecimal.valueOf(500.414).setScale(4)),
@@ -189,7 +189,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("select string type with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE ORDER BY NAME")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "akash"),
Map("NAME" -> "anubhav"),
@@ -206,7 +206,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("select DATE type with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT DATE FROM TESTDB.TESTTABLE ORDER BY DATE")
val expectedResult: List[Map[String, Any]] = List(Map("DATE" -> "2015-07-18"),
Map("DATE" -> "2015-07-23"),
@@ -228,7 +228,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.reverse.head("DATE") == null)
}
test("select int type with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT DISTINCT ID FROM TESTDB.TESTTABLE ORDER BY ID")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1),
Map("ID" -> 2),
@@ -247,7 +247,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test and filter clause with greater than expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>1234 AND ID>2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -274,7 +274,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test and filter clause with greater than equal to expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -315,7 +315,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.toString() equals expectedResult.toString())
}
test("test and filter clause with less than equal to expression") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS<=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -340,7 +340,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.toString() equals expectedResult.toString())
}
test("test equal to expression on decimal value") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID FROM TESTDB.TESTTABLE WHERE BONUS=1234.444")
@@ -349,7 +349,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult equals expectedResult)
}
test("test less than expression with and operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
"WHERE BONUS>1234 AND ID<2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
@@ -365,7 +365,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.toString().equals(expectedResult.toString()))
}
test("test the result for in clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME from testdb.testtable WHERE PHONETYPE IN('phone1848','phone706')")
val expectedResult: List[Map[String, Any]] = List(
Map("NAME" -> "geetika"),
@@ -375,7 +375,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("test the result for not in clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT NAME from testdb.testtable WHERE PHONETYPE NOT IN('phone1848','phone706')")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "anubhav"),
@@ -389,21 +389,21 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("test for null operator on date data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT ID FROM TESTDB.TESTTABLE WHERE DATE IS NULL")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 9),Map("ID" -> null))
assert(actualResult.equals(expectedResult))
}
test("test for not null operator on date data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DATE IS NOT NULL AND ID=9")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"))
assert(actualResult.equals(expectedResult))
}
test("test for not null operator on timestamp type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DOB IS NOT NULL AND ID=9")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"),
Map("NAME" -> "jitesh"))
@@ -411,14 +411,14 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test for null operator on timestamp type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DOB IS NULL AND ID=1")
val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "anubhav"))
assert(actualResult.equals(expectedResult))
}
test("test the result for short datatype with order by clause") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT DISTINCT SHORTFIELD from testdb.testtable ORDER BY SHORTFIELD ")
val expectedResult: List[Map[String, Any]] = List(Map("SHORTFIELD" -> 1),
@@ -433,7 +433,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("test the result for short datatype in clause where field is null") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID from testdb.testtable WHERE SHORTFIELD IS NULL ORDER BY SHORTFIELD ")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 7),Map("ID" -> null))
@@ -441,7 +441,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
assert(actualResult.equals(expectedResult))
}
test("test the result for short datatype with greater than operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID from testdb.testtable WHERE SHORTFIELD>11 ")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 6), Map("ID" -> 9))
@@ -450,7 +450,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test longDecimal type of presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID from testdb.testtable WHERE bonus = DECIMAL '1234.5555'")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 2))
@@ -459,7 +459,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test shortDecimal type of presto") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery(
"SELECT ID from testdb.testtable WHERE monthlyBonus = 15.13")
val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 2))
@@ -468,42 +468,42 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test timestamp datatype using cast operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT NAME AS RESULT FROM TESTDB.TESTTABLE WHERE DOB = CAST('2016-04-14 15:00:09' AS TIMESTAMP)")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "jatin"))
assert(actualResult.equals(expectedResult))
}
test("test timestamp datatype using cast and in operator") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT ID AS RESULT FROM TESTDB.TESTTABLE WHERE DOB in (cast('2016-04-14 " +
"15:00:09' as timestamp),cast('2015-10-07' as timestamp),cast('2015-10-07 01:00:03' as timestamp))")
val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "2"))
assert(actualResult.toString() equals expectedResult.toString())
}
test("test the boolean data type") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT isCurrentEmployee AS RESULT FROM TESTDB.TESTTABLE WHERE ID=1")
assert(actualResult.head("RESULT").toString.toBoolean)
}
test("test the boolean data type for null value") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT id AS RESULT FROM TESTDB.TESTTABLE WHERE isCurrentEmployee is null")
assert(actualResult.head("RESULT").toString.toInt==2)
}
test("test the boolean data type for not null value with filter ") {
- val actualResult: List[Map[String, Any]] = PrestoServer
+ val actualResult: List[Map[String, Any]] = prestoServer
.executeQuery("SELECT id AS RESULT FROM TESTDB.TESTTABLE WHERE isCurrentEmployee is NOT null AND ID>8")
assert(actualResult.head("RESULT").toString.toInt==9)
}
test("test the is null operator when null is included in string data type dictionary_include"){
// See CARBONDATA-2155
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SERIALNAME FROM TESTDB.TESTTABLE WHERE SERIALNAME IS NULL")
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT SERIALNAME FROM TESTDB.TESTTABLE WHERE SERIALNAME IS NULL")
assert(actualResult equals List(Map("SERIALNAME" -> null)))
}
test("test the min function when null is included in string data type with dictionary_include"){
// See CARBONDATA-2152
- val actualResult = PrestoServer.executeQuery("SELECT MIN(SERIALNAME) FROM TESTDB.TESTTABLE")
+ val actualResult = prestoServer.executeQuery("SELECT MIN(SERIALNAME) FROM TESTDB.TESTTABLE")
val expectedResult = List(Map("_col0" -> "ASD14875"))
assert(actualResult.equals(expectedResult))
@@ -511,11 +511,11 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
test("test the show schemas result"){
- val actualResult = PrestoServer.executeQuery("SHOW SCHEMAS")
+ val actualResult = prestoServer.executeQuery("SHOW SCHEMAS")
assert(actualResult.equals(List(Map("Schema" -> "information_schema"), Map("Schema" -> "testdb"))))
}
test("test the show tables"){
- val actualResult = PrestoServer.executeQuery("SHOW TABLES")
+ val actualResult = prestoServer.executeQuery("SHOW TABLES")
assert(actualResult.equals(List(Map("Table" -> "testtable"))))
}
@@ -531,7 +531,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the OR operator on same column"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT BONUS FROM TESTDB.TESTTABLE WHERE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT BONUS FROM TESTDB.TESTTABLE WHERE" +
" BONUS < 600 OR BONUS > 5000 ORDER BY BONUS")
val expectedResult: List[Map[String, Any]] = List(
Map("BONUS" -> java.math.BigDecimal.valueOf(500.4140).setScale(4)),
@@ -544,7 +544,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the AND, OR operator on same column"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
" SHORTFIELD > 4 AND (SHORTFIELD < 10 or SHORTFIELD > 15) ORDER BY SHORTFIELD")
val expectedResult: List[Map[String, Any]] = List(
Map("SHORTFIELD" -> 8),
@@ -553,7 +553,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the OR operator with multiple AND on same column"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
" (SHORTFIELD > 1 AND SHORTFIELD < 5) OR (SHORTFIELD > 10 AND SHORTFIELD < 15) ORDER BY SHORTFIELD")
val expectedResult: List[Map[String, Any]] = List(
Map("SHORTFIELD" -> 4),
@@ -563,7 +563,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the OR, AND operator with on Different column"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
" ID < 7 AND (SHORTFIELD < 5 OR SHORTFIELD > 15) ORDER BY SHORTFIELD")
val expectedResult: List[Map[String, Any]] = List(
Map("SHORTFIELD" -> 4),
@@ -572,7 +572,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
}
test("test the Timestamp greaterthan expression"){
- val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT DOB FROM TESTDB.TESTTABLE" +
+ val actualResult: List[Map[String, Any]] = prestoServer.executeQuery("SELECT DOB FROM TESTDB.TESTTABLE" +
" WHERE DOB > timestamp '2016-01-01 00:00:00.0' order by DOB")
val expectedResult: List[Map[String, Any]] = List(
Map("DOB" -> new Timestamp(new java.util.Date(2016-1900,1-1,14,15,7,9).getTime)),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/992b8f40/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
new file mode 100644
index 0000000..4c2f51f
--- /dev/null
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoTestNonTransactionalTableFiles.scala
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.presto.integrationtest
+
+import java.io.File
+import java.sql.SQLException
+import java.util
+
+import org.apache.commons.io.FileUtils
+import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+import org.apache.carbondata.presto.server.PrestoServer
+import org.apache.carbondata.sdk.file.{CarbonWriter, Field, Schema}
+
+
+class PrestoTestNonTransactionalTableFiles extends FunSuiteLike with BeforeAndAfterAll {
+
+ private val logger = LogServiceFactory
+ .getLogService(classOf[PrestoTestNonTransactionalTableFiles].getCanonicalName)
+
+ private val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ private val storePath = s"$rootPath/integration/presto/target/store"
+ private val systemPath = s"$rootPath/integration/presto/target/system"
+ private val writerPath = storePath + "/sdk_output/files"
+ private val prestoServer = new PrestoServer
+
+ override def beforeAll: Unit = {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION,
+ systemPath)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME,
+ "Presto")
+ prestoServer.startServer(storePath, "sdk_output")
+ }
+
+ override def afterAll(): Unit = {
+ prestoServer.stopServer()
+ }
+
+ def buildTestDataSingleFile(): Any = {
+ FileUtils.deleteDirectory(new File(writerPath))
+ buildTestData(3, null)
+ }
+
+ def buildTestDataMultipleFiles(): Any = {
+ FileUtils.deleteDirectory(new File(writerPath))
+ buildTestData(1000000, null)
+ }
+
+ def buildTestData(rows: Int, options: util.Map[String, String]): Any = {
+ buildTestData(rows, options, List("name"))
+ }
+
+ // prepare sdk writer output
+ def buildTestData(rows: Int,
+ options: util.Map[String, String],
+ sortColumns: List[String]): Any = {
+ val schema = new StringBuilder()
+ .append("[ \n")
+ .append(" {\"NaMe\":\"string\"},\n")
+ .append(" {\"age\":\"int\"},\n")
+ .append(" {\"height\":\"double\"}\n")
+ .append("]")
+ .toString()
+
+ try {
+ val builder = CarbonWriter.builder()
+ val writer =
+ if (options != null) {
+ builder.outputPath(writerPath)
+ .sortBy(sortColumns.toArray)
+ .uniqueIdentifier(
+ System.currentTimeMillis).withBlockSize(2).withLoadOptions(options)
+ .withCsvInput(Schema.parseJson(schema)).writtenBy("TestNonTransactionalCarbonTable")
+ .build()
+ } else {
+ builder.outputPath(writerPath)
+ .sortBy(sortColumns.toArray)
+ .uniqueIdentifier(
+ System.currentTimeMillis).withBlockSize(2)
+ .withCsvInput(Schema.parseJson(schema)).writtenBy("TestNonTransactionalCarbonTable")
+ .build()
+ }
+ var i = 0
+ while (i < rows) {
+ if ((options != null) && (i < 3)) {
+ // writing a bad record
+ writer.write(Array[String]("robot" + i, String.valueOf(i.toDouble / 2), "robot"))
+ } else {
+ writer
+ .write(Array[String]("robot" + i, String.valueOf(i), String.valueOf(i.toDouble / 2)))
+ }
+ i += 1
+ }
+ if (options != null) {
+ //Keep one valid record. else carbon data file will not generate
+ writer.write(Array[String]("robot" + i, String.valueOf(i), String.valueOf(i.toDouble / 2)))
+ }
+ writer.close()
+ } catch {
+ case ex: Throwable => throw new RuntimeException(ex)
+ }
+ }
+
+ // prepare sdk writer output with other schema
+ def buildTestDataOtherDataType(rows: Int, sortColumns: Array[String]): Any = {
+ val fields: Array[Field] = new Array[Field](3)
+ // same column name, but name as boolean type
+ fields(0) = new Field("name", DataTypes.BOOLEAN)
+ fields(1) = new Field("age", DataTypes.INT)
+ fields(2) = new Field("height", DataTypes.DOUBLE)
+
+ try {
+ val builder = CarbonWriter.builder()
+ val writer =
+ builder.outputPath(writerPath)
+ .uniqueIdentifier(System.currentTimeMillis()).withBlockSize(2).sortBy(sortColumns)
+ .withCsvInput(new Schema(fields)).writtenBy("TestNonTransactionalCarbonTable").build()
+ var i = 0
+ while (i < rows) {
+ writer.write(Array[String]("true", String.valueOf(i), String.valueOf(i.toDouble / 2)))
+ i += 1
+ }
+ writer.close()
+ } catch {
+ case ex: Throwable => throw new RuntimeException(ex)
+ }
+ }
+
+ def cleanTestData(): Unit = {
+ FileUtils.deleteDirectory(new File(writerPath))
+ }
+
+ def deleteFile(path: String, extension: String): Unit = {
+ val file: CarbonFile = FileFactory
+ .getCarbonFile(path, FileFactory.getFileType(path))
+
+ for (eachDir <- file.listFiles) {
+ if (!eachDir.isDirectory) {
+ if (eachDir.getName.endsWith(extension)) {
+ CarbonUtil.deleteFoldersAndFilesSilent(eachDir)
+ }
+ } else {
+ deleteFile(eachDir.getPath, extension)
+ }
+ }
+ }
+
+ test("test show schemas") {
+ buildTestDataSingleFile()
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("show schemas ")
+ assert(actualResult
+ .equals(List(Map("Schema" -> "information_schema"),
+ Map("Schema" -> "sdk_output"),
+ Map("Schema" -> "testdb"))))
+ cleanTestData()
+ }
+
+ test("test show tables") {
+ buildTestDataSingleFile()
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("show tables")
+ assert(actualResult
+ .equals(List(Map("Table" -> "files"))))
+ cleanTestData()
+ }
+
+ test("test read sdk output files") {
+ buildTestDataSingleFile()
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("SELECT COUNT(*) AS RESULT FROM files ")
+ val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 3))
+ assert(actualResult.equals(expectedResult))
+ cleanTestData()
+ }
+
+ test("test read multiple carbondata and index files") {
+ buildTestDataMultipleFiles()
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("SELECT COUNT(*) AS RESULT FROM files ")
+ val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 1000000))
+ assert(actualResult.equals(expectedResult))
+ cleanTestData()
+ }
+
+ test("test reading different schema") {
+ buildTestDataSingleFile()
+ buildTestDataOtherDataType(3, null)
+ val exception =
+ intercept[SQLException] {
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("select count(*) as RESULT from files ")
+ }
+ assert(exception.getMessage()
+ .contains("All the files doesn't have same schema"))
+ cleanTestData()
+ }
+
+ test("test reading without carbon index file") {
+ buildTestDataSingleFile()
+ deleteFile(writerPath, CarbonCommonConstants.UPDATE_INDEX_FILE_EXT)
+ val exception =
+ intercept[SQLException] {
+ val actualResult: List[Map[String, Any]] = prestoServer
+ .executeQuery("select * from files ")
+ }
+ assert(exception.getMessage()
+ .contains("No Index files are present in the table location"))
+ cleanTestData()
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/992b8f40/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
index 1b1accf..5c0c40a 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
@@ -32,7 +32,6 @@ import com.facebook.presto.tests.DistributedQueryRunner
import com.google.common.collect.ImmutableMap
import org.slf4j.{Logger, LoggerFactory}
-import org.apache.carbondata.core.util.ThreadLocalSessionInfo
import org.apache.carbondata.presto.CarbondataPlugin
class PrestoServer {
@@ -46,6 +45,7 @@ class PrestoServer {
val prestoProperties: util.Map[String, String] = Map(("http-server.http.port", "8086")).asJava
createSession
lazy val queryRunner = new DistributedQueryRunner(createSession, 4, prestoProperties)
+ var dbName : String = null
/**
@@ -53,7 +53,24 @@ class PrestoServer {
*
* @param carbonStorePath the store path of carbon
*/
- def startServer(carbonStorePath: String) = {
+ def startServer(carbonStorePath: String): Unit = {
+
+ logger.info("======== STARTING PRESTO SERVER ========")
+ val queryRunner: DistributedQueryRunner = createQueryRunner(
+ prestoProperties, carbonStorePath)
+
+ logger.info("STARTED SERVER AT :" + queryRunner.getCoordinator.getBaseUrl)
+ }
+
+ /**
+ * start the presto server
+ *
+ * @param carbonStorePath the store path of carbon
+ * @param dbName the database name , if not a default database
+ */
+ def startServer(carbonStorePath: String, dbName: String): Unit = {
+
+ this.dbName = dbName
logger.info("======== STARTING PRESTO SERVER ========")
val queryRunner: DistributedQueryRunner = createQueryRunner(
@@ -99,7 +116,7 @@ class PrestoServer {
def executeQuery(query: String): List[Map[String, Any]] = {
Try {
- val conn: Connection = createJdbcConnection
+ val conn: Connection = createJdbcConnection(dbName)
logger.info(s"***** executing the query ***** \n $query")
val statement = conn.createStatement()
val result: ResultSet = statement.executeQuery(query)
@@ -117,13 +134,17 @@ class PrestoServer {
*
* @return
*/
- private def createJdbcConnection: Connection = {
+ private def createJdbcConnection(dbName: String) = {
val JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"
- val DB_URL = "jdbc:presto://localhost:8086/carbondata/testdb"
-
+ var DB_URL : String = null
+ if (dbName == null) {
+ DB_URL = "jdbc:presto://localhost:8086/carbondata/default"
+ } else {
+ DB_URL = "jdbc:presto://localhost:8086/carbondata/" + dbName
+ }
val properties = new Properties
// The database Credentials
- properties.setProperty("user", "test");
+ properties.setProperty("user", "test")
// STEP 2: Register JDBC driver
Class.forName(JDBC_DRIVER)
[15/21] carbondata git commit: [CARBONDATA-3002] Fix some spell error
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala b/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala
index b6667df..efaa191 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/TableLoader.scala
@@ -82,7 +82,7 @@ object TableLoader {
val spark = TableAPIUtil.spark(storePath, s"TableLoader: $dbName.$tableName")
- CarbonEnv.getInstance(spark).carbonMetastore.
+ CarbonEnv.getInstance(spark).carbonMetaStore.
checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
loadTable(spark, Option(dbName), tableName, inputPaths, map)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
index 759539b..47b6b67 100644
--- a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -164,10 +164,10 @@ class CarbonHiveSessionCatalog(
carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation): Boolean = {
var isRefreshed = false
val storePath = CarbonProperties.getStorePath
- carbonEnv.carbonMetastore.
+ carbonEnv.carbonMetaStore.
checkSchemasModifiedTimeAndReloadTable(identifier)
- val table = carbonEnv.carbonMetastore.getTableFromMetadataCache(
+ val table = carbonEnv.carbonMetaStore.getTableFromMetadataCache(
carbonDatasourceHadoopRelation.carbonTable.getDatabaseName,
carbonDatasourceHadoopRelation.carbonTable.getTableName)
if (table.isEmpty || (table.isDefined &&
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index ed5486b..bceb0fc 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -137,7 +137,7 @@ class AllDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll {
.config("spark.executor.heartbeatInterval", "600s")
.config("carbon.enable.vector.reader","false")
.getOrCreateCarbonSession(storeLocation, metastoredb)
- val catalog = CarbonEnv.getInstance(spark).carbonMetastore
+ val catalog = CarbonEnv.getInstance(spark).carbonMetaStore
sampleRelation = catalog.lookupRelation(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
"sample")(spark).asInstanceOf[CarbonRelation]
complexRelation = catalog.lookupRelation(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryLRUCacheTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryLRUCacheTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryLRUCacheTestCase.scala
index 245ee7c..3c50a18 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryLRUCacheTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/DictionaryLRUCacheTestCase.scala
@@ -39,7 +39,7 @@ class DictionaryLRUCacheTestCase extends Spark2QueryTest with BeforeAndAfterAll
var path : String = null
def checkDictionaryAccessCount(databaseName: String, tableName: String): Unit = {
- val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetastore
+ val carbonTable = CarbonEnv.getInstance(Spark2TestQueryExecutor.spark).carbonMetaStore
.lookupRelation(Option(databaseName), tableName)(Spark2TestQueryExecutor.spark)
.asInstanceOf[CarbonRelation].carbonTable
val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index 69248d6..9607bbc 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -132,7 +132,7 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
.config("spark.executor.heartbeatInterval", "600s")
.config("carbon.enable.vector.reader","false")
.getOrCreateCarbonSession(storeLocation, metastoredb)
- val catalog = CarbonEnv.getInstance(spark).carbonMetastore
+ val catalog = CarbonEnv.getInstance(spark).carbonMetaStore
extComplexRelation = catalog
.lookupRelation(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
"extComplextypes")(spark)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala
index e3c2d88..5096089 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOpName.scala
@@ -226,7 +226,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
assertResult("Streaming property value is incorrect")(msg.getMessage)
val identifier = new TableIdentifier("batch_table", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
var server: ServerSocket = null
try {
@@ -253,7 +253,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
// input source: file
test("streaming ingest from file source") {
val identifier = new TableIdentifier("stream_table_file", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdata").getCanonicalPath
// streaming ingest 10 rows
@@ -277,7 +277,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
test("test preaggregate table creation on streaming table without handoff") {
val identifier = new TableIdentifier("agg_table", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdatanew").getCanonicalPath
// streaming ingest 10 rows
@@ -297,7 +297,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
test("test if data is loaded into preaggregate after handoff is fired") {
createTable(tableName = "agg_table2", streaming = true, withBatchLoad = false)
val identifier = new TableIdentifier("agg_table2", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdatanew").getCanonicalPath
// streaming ingest 10 rows
@@ -358,7 +358,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
test("test whether data is loaded into preaggregate after handoff is fired") {
createTable(tableName = "agg_table2", streaming = true, withBatchLoad = false)
val identifier = new TableIdentifier("agg_table2", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdatanew").getCanonicalPath
// streaming ingest 10 rows
@@ -401,7 +401,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
test("test whether data is loaded into preaggregate before handoff is fired") {
createTable(tableName = "agg_table2", streaming = true, withBatchLoad = false)
val identifier = new TableIdentifier("agg_table2", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdatanew").getCanonicalPath
// streaming ingest 10 rows
@@ -438,7 +438,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists timeseries_table")
createTable(tableName = "timeseries_table", streaming = true, withBatchLoad = false)
val identifier = new TableIdentifier("timeseries_table", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdatanew").getCanonicalPath
val thread = createFileStreamingThread(spark, carbonTable, csvDataDir, intervalSecond = 1,
@@ -465,7 +465,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists timeseries_table")
createTable(tableName = "timeseries_table", streaming = true, withBatchLoad = false)
val identifier = new TableIdentifier("timeseries_table", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdatanew").getCanonicalPath
val thread = createFileStreamingThread(spark, carbonTable, csvDataDir, intervalSecond = 1,
@@ -549,7 +549,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
def loadData() {
val identifier = new TableIdentifier("agg_table2", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdatanew").getCanonicalPath
// streaming ingest 10 rows
@@ -567,7 +567,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists agg_table2")
createTable(tableName = "agg_table2", streaming = true, withBatchLoad = false)
val identifier = new TableIdentifier("agg_table2", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdata1").getCanonicalPath
generateCSVDataFile(spark, idStart = 10, rowNums = 5, csvDataDir)
@@ -598,7 +598,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
test("test if data is loaded in aggregate table after handoff is done for streaming table") {
createTable(tableName = "agg_table3", streaming = true, withBatchLoad = false)
val identifier = new TableIdentifier("agg_table3", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
val csvDataDir = new File("target/csvdatanew").getCanonicalPath
generateCSVDataFile(spark, idStart = 10, rowNums = 5, csvDataDir)
@@ -1621,7 +1621,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
test("block drop streaming table while streaming is in progress") {
val identifier = new TableIdentifier("stream_table_drop", Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
var server: ServerSocket = null
try {
@@ -2420,7 +2420,7 @@ class TestStreamingTableOpName extends QueryTest with BeforeAndAfterAll {
badRecordsPath: String = CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL
): Unit = {
val identifier = new TableIdentifier(tableName, Option("streaming"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
var server: ServerSocket = null
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
index 21cad07..5d806a3 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableWithRowParser.scala
@@ -816,7 +816,7 @@ class TestStreamingTableWithRowParser extends QueryTest with BeforeAndAfterAll {
autoHandoff: Boolean = CarbonCommonConstants.ENABLE_AUTO_HANDOFF_DEFAULT.toBoolean
): Unit = {
val identifier = new TableIdentifier(tableName, Option("streaming1"))
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetaStore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
var server: ServerSocket = null
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala
index a0c801a..e4e7d92 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala
@@ -67,7 +67,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'a',1,'aa','aaa'")
backUpData(dblocation, "carbontable")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
sql("refresh table carbontable")
checkAnswer(sql("select count(*) from carbontable"), Row(1))
@@ -83,7 +83,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'a',1,'aa','aaa'")
backUpData(dblocation, "carbontable")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
sql("refresh table carbontable")
checkAnswer(sql("select count(*) from carbontable"), Row(1))
@@ -103,7 +103,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
backUpData(dblocation, "carbontable")
backUpData(dblocation, "carbontable_preagg1")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
restoreData(dblocation, "carbontable_preagg1")
sql("refresh table carbontable")
@@ -126,7 +126,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
backUpData(dblocation, "carbontable")
backUpData(dblocation, "carbontable_preagg1")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
restoreData(dblocation, "carbontable_preagg1")
sql("refresh table carbontable")
@@ -149,7 +149,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
backUpData(dblocation, "carbontable")
backUpData(dblocation, "carbontable_preagg1")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
intercept[ProcessMetaDataException] {
sql("refresh table carbontable")
@@ -169,7 +169,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'b',1,'bb','bbb'")
backUpData(dblocation, "carbontable")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
sql("refresh table carbontable")
// update operation
@@ -195,7 +195,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$testData' into table automerge")
backUpData(dblocation, "automerge")
sql("drop table automerge")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "automerge")
sql("refresh table automerge")
// update operation
@@ -216,7 +216,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'b',1,'bb','bbb'")
backUpData(dblocation, "carbontable")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
sql("refresh table carbontable")
// delete operation
@@ -238,7 +238,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'b',1,'bb','bbb'")
backUpData(dblocation, "carbontable")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
sql("refresh table carbontable")
sql("Alter table carbontable add columns(c4 string) " +
@@ -260,7 +260,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'b',1,'bb','bbb'")
backUpData(dblocation, "carbontable")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
sql("refresh table carbontable")
sql("Alter table carbontable change c2 c2 long")
@@ -281,7 +281,7 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("insert into carbontable select 'b',1,'bb','bbb'")
backUpData(dblocation, "carbontable")
sql("drop table carbontable")
- if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
+ if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
sql("refresh table carbontable")
sql("Alter table carbontable drop columns(c2)")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
index d3501c7..54dc2d4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
@@ -116,7 +116,7 @@ public class CarbonDataLoadConfiguration {
private boolean carbonTransactionalTable;
/**
- * Flder path to where data should be written for this load.
+ * Folder path to where data should be written for this load.
*/
private String dataWritePath;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index aecc52e..71d61db 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -217,7 +217,7 @@ public class CarbonLoadModel implements Serializable {
private boolean isJsonFileLoad;
/**
- * Flder path to where data should be written for this load.
+ * Folder path to where data should be written for this load.
*/
private String dataWritePath;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/313ba0ea/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 64fcaa2..7688415 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -551,14 +551,14 @@ public final class CarbonLoaderUtil {
* This method will divide the blocks among the nodes as per the data locality
*
* @param blockInfos blocks
- * @param noOfNodesInput -1 if number of nodes has to be decided
+ * @param numOfNodesInput -1 if number of nodes has to be decided
* based on block location information
* @param blockAssignmentStrategy strategy used to assign blocks
* @param expectedMinSizePerNode the property load_min_size_inmb specified by the user
* @return a map that maps node to blocks
*/
public static Map<String, List<Distributable>> nodeBlockMapping(
- List<Distributable> blockInfos, int noOfNodesInput, List<String> activeNodes,
+ List<Distributable> blockInfos, int numOfNodesInput, List<String> activeNodes,
BlockAssignmentStrategy blockAssignmentStrategy, String expectedMinSizePerNode) {
ArrayList<NodeMultiBlockRelation> rtnNode2Blocks = new ArrayList<>();
@@ -569,9 +569,9 @@ public final class CarbonLoaderUtil {
nodes.add(relation.getNode());
}
- int noofNodes = (-1 == noOfNodesInput) ? nodes.size() : noOfNodesInput;
+ int numOfNodes = (-1 == numOfNodesInput) ? nodes.size() : numOfNodesInput;
if (null != activeNodes) {
- noofNodes = activeNodes.size();
+ numOfNodes = activeNodes.size();
}
// calculate the average expected size for each node
@@ -579,7 +579,7 @@ public final class CarbonLoaderUtil {
long totalFileSize = 0;
if (BlockAssignmentStrategy.BLOCK_NUM_FIRST == blockAssignmentStrategy) {
if (blockInfos.size() > 0) {
- sizePerNode = blockInfos.size() / noofNodes;
+ sizePerNode = blockInfos.size() / numOfNodes;
}
sizePerNode = sizePerNode <= 0 ? 1 : sizePerNode;
} else if (BlockAssignmentStrategy.BLOCK_SIZE_FIRST == blockAssignmentStrategy
@@ -587,7 +587,7 @@ public final class CarbonLoaderUtil {
for (Distributable blockInfo : uniqueBlocks) {
totalFileSize += ((TableBlockInfo) blockInfo).getBlockLength();
}
- sizePerNode = totalFileSize / noofNodes;
+ sizePerNode = totalFileSize / numOfNodes;
}
// if enable to control the minimum amount of input data for each node
@@ -1152,7 +1152,7 @@ public final class CarbonLoaderUtil {
* @return
* @throws IOException
*/
- public static String mergeIndexFilesinPartitionedSegment(CarbonTable table, String segmentId,
+ public static String mergeIndexFilesInPartitionedSegment(CarbonTable table, String segmentId,
String uuid) throws IOException {
String tablePath = table.getTablePath();
return new CarbonIndexFileMergeWriter(table)
[20/21] carbondata git commit: [CARBONDATA-3175]Fix Testcase failures
in complex delimiters
Posted by ra...@apache.org.
[CARBONDATA-3175]Fix Testcase failures in complex delimiters
Changed the complex delimiter in testcases
This closes #2989
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/951a7811
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/951a7811
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/951a7811
Branch: refs/heads/branch-1.5
Commit: 951a7811a416ea4c000a804d56ae2ac3433736c8
Parents: ef78720
Author: Indhumathi27 <in...@gmail.com>
Authored: Fri Dec 14 16:26:00 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:34 2018 +0530
----------------------------------------------------------------------
.../sdv/generated/ComplexDataTypeTestCase.scala | 35 ++++++++++----------
1 file changed, 18 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/951a7811/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala
index 723c4f7..6c2e271 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/ComplexDataTypeTestCase.scala
@@ -74,7 +74,7 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
sql(
"create table test(person struct<detail:struct<id:int,name:string,height:double," +
"status:boolean,dob:date,dobt:timestamp>>) stored by 'carbondata'")
- sql("insert into test values('1:abc:4.30:true:2017/08/09:2017/08/09')")
+ sql("insert into test values('1\002abc\0024.30\002true\0022017/08/09\0022017/08/09')")
checkAnswer(sql("select * from test"),
Seq(Row(Row(Row(1, "abc", 4.3, true, java.sql.Date.valueOf("2017-08-09"),
Timestamp.valueOf("2017-08-09 00:00:00.0"))))))
@@ -83,8 +83,8 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"create table test(p1 array<int>,p2 array<string>,p3 array<double>,p4 array<boolean>,p5 " +
"array<date>,p6 array<timestamp>) stored by 'carbondata'")
sql(
- "insert into test values('1$2$3','abc$def$mno','4.30$4.60$5.20','true$true$false'," +
- "'2017/08/09$2017/08/09$2017/07/07','2017/08/09$2017/08/09$2017/07/07')")
+ "insert into test values('1\0012\0013','abc\001def\001mno','4.30\0014.60\0015.20','true\001true\001false'," +
+ "'2017/08/09\0012017/08/09\0012017/07/07','2017/08/09\0012017/08/09\0012017/07/07')")
checkAnswer(sql("select * from test"),
Seq(Row(mutable.WrappedArray.make(Array(1, 2, 3)),
mutable.WrappedArray.make(Array("abc", "def", "mno")),
@@ -142,7 +142,7 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"create table test(person struct<detail:struct<id:int,name:string,height:double," +
"status:boolean,dob:date,dobt:timestamp>>) stored by 'carbondata' tblproperties" +
"('dictionary_include'='person')")
- sql("insert into test values('1:abc:4.30:true:2017/08/09:2017/08/09')")
+ sql("insert into test values('1\002abc\0024.30\002true\0022017/08/09\0022017/08/09')")
checkAnswer(sql("select * from test"),
Seq(Row(Row(Row(1,
"abc", 4.3, true, java.sql.Date.valueOf("2017-08-09"),
@@ -153,8 +153,8 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"array<date>,p6 array<timestamp>) stored by 'carbondata' tblproperties" +
"('dictionary_include'='p1,p2,p3,p4,p5,p6')")
sql(
- "insert into test values('1$2$3','abc$def$mno','4.30$4.60$5.20','true$true$false'," +
- "'2017/08/09$2017/08/09$2017/07/07','2017/08/09$2017/08/09$2017/07/07')")
+ "insert into test values('1\0012\0013','abc\001def\001mno','4.30\0014.60\0015.20','true\001true\001false'," +
+ "'2017/08/09\0012017/08/09\0012017/07/07','2017/08/09\0012017/08/09\0012017/07/07')")
checkAnswer(sql("select * from test"),
Seq(Row(mutable.WrappedArray.make(Array(1, 2, 3)),
mutable.WrappedArray.make(Array("abc", "def", "mno")),
@@ -206,9 +206,9 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"h:string,i:int>,j:int>) stored " +
"by " +
"'carbondata'")
- sql("insert into complexcarbontable values(1,'1$abc$2$efg$3:mno:4$5')")
- sql("insert into complexcarbontable values(2,'1$abc$2$efg$3:mno:4$5')")
- sql("insert into complexcarbontable values(3,'1$abc$2$efg$3:mno:4$5')")
+ sql("insert into complexcarbontable values(1,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
+ sql("insert into complexcarbontable values(2,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
+ sql("insert into complexcarbontable values(3,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
checkAnswer(sql("select a.b from complexcarbontable"), Seq(Row(1), Row(1), Row(1)))
checkAnswer(sql("select a.c from complexcarbontable"), Seq(Row("abc"), Row("abc"), Row("abc")))
checkAnswer(sql("select a.d from complexcarbontable"), Seq(Row(2), Row(2), Row(2)))
@@ -248,9 +248,9 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
test("test Complex_DataType-006") {
sql("DROP TABLE IF EXISTS test")
sql("create table test(id int,a struct<b:int,c:int>) stored by 'carbondata'")
- sql("insert into test values(1,'2$3')")
- sql("insert into test values(3,'5$3')")
- sql("insert into test values(2,'4$5')")
+ sql("insert into test values(1,'2\0013')")
+ sql("insert into test values(3,'5\0013')")
+ sql("insert into test values(2,'4\0015')")
checkAnswer(sql("select a.b from test where id=3"), Seq(Row(5)))
checkAnswer(sql("select a.b from test where a.c!=3"), Seq(Row(4)))
checkAnswer(sql("select a.b from test where a.c=3"), Seq(Row(5), Row(2)))
@@ -294,7 +294,7 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"create table complexcarbontable(roll int, student struct<id:int,name:string," +
"marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into complexcarbontable values(1,'null$abc$null:null:null')")
+ sql("insert into complexcarbontable values(1,'null\001abc\001null\002null\002null')")
checkAnswer(sql("select * from complexcarbontable"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -308,14 +308,14 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
"stored by 'carbondata'")
sql(
"insert into complexcarbontable values" +
- "('14.35:400000.35:1.7976931348623157$67890985.888:65.5656:200')," +
- "('20.25:50000.25:4.945464565654656546546546324$10000000:300000:3000')")
+ "('14.35\002400000.35\0021.7976931348623157\00167890985.888\00265.5656\002200')," +
+ "('20.25\00250000.25\0024.945464565654656546546546324\00110000000\002300000\0023000')")
checkExistence(sql("select * from complexcarbontable"), true, "1.0E7,300000.0,3000.0")
sql("Drop table if exists complexcarbontable")
sql(
"create table complexcarbontable(struct_arr struct<array_db1:array<double>>) stored by " +
"'carbondata'")
- sql("insert into complexcarbontable values('5555555.9559:12345678991234567:3444.999')")
+ sql("insert into complexcarbontable values('5555555.9559\00212345678991234567\0023444.999')")
checkExistence(sql("select * from complexcarbontable"),
true,
"5555555.9559, 1.2345678991234568E16, 3444.999")
@@ -361,7 +361,8 @@ class ComplexDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
""".stripMargin
val pschema = org.apache.avro.Schema.parse(mySchema)
val records = jsonToAvro(jsonvalue, mySchema)
- val writer = CarbonWriter.builder().outputPath(writerPath).buildWriterForAvroInput(pschema)
+ val writer = CarbonWriter.builder().outputPath(writerPath).withAvroInput(pschema)
+ .writtenBy("ComplexDataTypeTestCase").build()
writer.write(records)
writer.close()
sql("DROP TABLE IF EXISTS sdkOutputTable")
[03/21] carbondata git commit: [CARBONDATA-3154] Fix spark-2.1 test
error
Posted by ra...@apache.org.
[CARBONDATA-3154] Fix spark-2.1 test error
Spark2.2.1 supports location, but Spark2.1.0 doesn't support location,
supports options(path 'your file path').
So we should change location to options(path ... ) and
create new directory before use "create table" in spark2.1.0.
This closes #2981
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3524f51d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3524f51d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3524f51d
Branch: refs/heads/branch-1.5
Commit: 3524f51df087059f1fa7ccd87f2082e4d2d36c20
Parents: 442e244
Author: xubo245 <xu...@huawei.com>
Authored: Sat Dec 8 00:01:43 2018 +0800
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:50:23 2018 +0530
----------------------------------------------------------------------
.../datasource/SparkCarbonDataSourceTest.scala | 112 +++++++++++++++----
1 file changed, 91 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3524f51d/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
index c5d6a8c..470e0bf 100644
--- a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
+++ b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
@@ -19,13 +19,14 @@ package org.apache.spark.sql.carbondata.datasource
import java.io.File
import java.util
-import java.util.Arrays
import scala.collection.JavaConverters._
import scala.collection.mutable
+import org.apache.hadoop.fs.permission.{FsAction, FsPermission}
import org.apache.spark.sql.{AnalysisException, Row}
import org.apache.spark.sql.carbondata.datasource.TestUtil._
+import org.apache.spark.util.SparkUtil
import org.scalatest.{BeforeAndAfterAll, FunSuite}
import org.apache.carbondata.core.datamap.DataMapStoreManager
@@ -998,9 +999,19 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
i += 1
}
writer.close()
- spark.sql("create table complextable (stringfield string, structfield struct<bytefield: " +
- "byte, floatfield: float>) " +
- s"using carbon location '$path'")
+ if (SparkUtil.isSparkVersionEqualTo("2.1")) {
+ if (!FileFactory.isFileExist(path)) {
+ FileFactory.createDirectoryAndSetPermission(path,
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ }
+ spark.sql("create table complextable (stringfield string, structfield struct<bytefield: " +
+ "byte, floatfield: float>) " +
+ s"using carbon options(path '$path')")
+ } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
+ spark.sql("create table complextable (stringfield string, structfield struct<bytefield: " +
+ "byte, floatfield: float>) " +
+ s"using carbon location '$path'")
+ }
} catch {
case ex: Exception => throw new RuntimeException(ex)
case _ => None
@@ -1052,13 +1063,33 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
writer.close()
spark.sql("drop table if exists sorted_par")
spark.sql("drop table if exists sort_table")
- spark.sql(s"create table sort_table (age int, height double, name string, address string," +
- s" salary long, bytefield byte) using carbon location '$path'")
- FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(s"$warehouse1/../warehouse2"))
- spark.sql(s"create table sorted_par(age int, height double, name string, address " +
- s"string," +
- s"salary long, bytefield byte) using parquet location " +
- s"'$warehouse1/../warehouse2'")
+ val path2 = s"$warehouse1/../warehouse2";
+ if (SparkUtil.isSparkVersionEqualTo("2.1")) {
+ if (!FileFactory.isFileExist(path)) {
+ FileFactory.createDirectoryAndSetPermission(path,
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ }
+ spark.sql(s"create table sort_table (age int, height double, name string, address string," +
+ s" salary long, bytefield byte) using carbon options(path '$path')")
+ FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(s"$warehouse1/../warehouse2"))
+ if (!FileFactory.isFileExist(path2)) {
+ FileFactory.createDirectoryAndSetPermission(path2,
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ }
+ spark.sql(s"create table sorted_par(age int, height double, name string, address " +
+ s"string," +
+ s"salary long, bytefield byte) using parquet options(path " +
+ s"'$path2')")
+ } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
+ spark.sql(s"create table sort_table (age int, height double, name string, address string," +
+ s" salary long, bytefield byte) using carbon location '$path'")
+ FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(s"$warehouse1/../warehouse2"))
+ spark.sql(s"create table sorted_par(age int, height double, name string, address " +
+ s"string," +
+ s"salary long, bytefield byte) using parquet location " +
+ s"'$warehouse1/../warehouse2'")
+ }
+
(0 to 10).foreach {
i =>
spark.sql(s"insert into sorted_par select '$i', ${ i.toDouble / 2 }, 'name$i', " +
@@ -1098,10 +1129,21 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
i += 1
}
writer.close()
- spark.sql(s"create table complextable (stringfield string, bytearray " +
- s"array<byte>, floatarray array<float>) using carbon " +
- s"location " +
- s"'$path'")
+ if (SparkUtil.isSparkVersionEqualTo("2.1")) {
+ if (!FileFactory.isFileExist(path)) {
+ FileFactory.createDirectoryAndSetPermission(path,
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ }
+ spark.sql(s"create table complextable (stringfield string, bytearray " +
+ s"array<byte>, floatarray array<float>) using carbon " +
+ s"options( path " +
+ s"'$path')")
+ } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
+ spark.sql(s"create table complextable (stringfield string, bytearray " +
+ s"array<byte>, floatarray array<float>) using carbon " +
+ s"location " +
+ s"'$path'")
+ }
} catch {
case ex: Exception => throw new RuntimeException(ex)
case _ => None
@@ -1123,9 +1165,20 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
private def createParquetTable {
val path = FileFactory.getUpdatedFilePath(s"$warehouse1/../warehouse2")
FileFactory.deleteAllCarbonFilesOfDir(FileFactory.getCarbonFile(s"$path"))
- spark.sql(s"create table par_table(male boolean, age int, height double, name string, address " +
- s"string," +
- s"salary long, floatField float, bytefield byte) using parquet location '$path'")
+ if (SparkUtil.isSparkVersionEqualTo("2.1")) {
+ if (!FileFactory.isFileExist(path)) {
+ FileFactory.createDirectoryAndSetPermission(path,
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ }
+ spark.sql(s"create table par_table(male boolean, age int, height double, name string, address " +
+ s"string," +
+ s"salary long, floatField float, bytefield byte) using parquet options(path '$path')")
+ } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
+ spark.sql(s"create table par_table(male boolean, age int, height double, name string, address " +
+ s"string," +
+ s"salary long, floatField float, bytefield byte) using parquet location '$path'")
+ }
+
(0 to 10).foreach {
i => spark.sql(s"insert into par_table select 'true','$i', ${i.toDouble / 2}, 'name$i', " +
s"'address$i', ${i*100}, $i.$i, '$i'")
@@ -1237,7 +1290,15 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
val rowCount = 3
buildStructSchemaWithNestedArrayOfMapTypeAsValue(writerPath, rowCount)
spark.sql("drop table if exists carbon_external")
- spark.sql(s"create table carbon_external using carbon location '$writerPath'")
+ if (SparkUtil.isSparkVersionEqualTo("2.1")) {
+ if (!FileFactory.isFileExist(writerPath)) {
+ FileFactory.createDirectoryAndSetPermission(writerPath,
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ }
+ spark.sql(s"create table carbon_external using carbon options(path '$writerPath')")
+ } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
+ spark.sql(s"create table carbon_external using carbon location '$writerPath'")
+ }
assert(spark.sql("select * from carbon_external").count() == rowCount)
spark.sql("drop table if exists carbon_external")
}
@@ -1268,8 +1329,17 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
i += 1
}
writer.close()
- spark.sql(s"create table multi_page (a string, b float, c byte) using carbon location " +
- s"'$path'")
+ if (SparkUtil.isSparkVersionEqualTo("2.1")) {
+ if (!FileFactory.isFileExist(path)) {
+ FileFactory.createDirectoryAndSetPermission(path,
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ }
+ spark.sql(s"create table multi_page (a string, b float, c byte) using carbon options(path " +
+ s"'$path')")
+ } else if (SparkUtil.isSparkVersionXandAbove("2.2")) {
+ spark.sql(s"create table multi_page (a string, b float, c byte) using carbon location " +
+ s"'$path'")
+ }
assert(spark.sql("select * from multi_page").count() == 33000)
} catch {
case ex: Exception => throw new RuntimeException(ex)
[18/21] carbondata git commit: [CARBONDATA-3153] Complex delimiters
change
Posted by ra...@apache.org.
[CARBONDATA-3153] Complex delimiters change
Changed the two Complex Delimiters used to '\001' and '\002'.
This closes #2979
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3c597215
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3c597215
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3c597215
Branch: refs/heads/branch-1.5
Commit: 3c597215b96edb880041c4d651e5dfa665f4d2bb
Parents: 992b8f4
Author: manishnalla1994 <ma...@gmail.com>
Authored: Fri Dec 7 14:55:58 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:34 2018 +0530
----------------------------------------------------------------------
conf/dataload.properties.template | 4 +-
docs/dml-of-carbondata.md | 12 +-
.../hadoop/api/CarbonTableOutputFormat.java | 2 +-
.../complexType/TestAdaptiveComplexType.scala | 104 ++++----
.../TestAdaptiveEncodingForNullValues.scala | 22 +-
.../complexType/TestCompactionComplexType.scala | 250 +++++++++----------
.../complexType/TestComplexDataType.scala | 63 ++---
.../TestComplexTypeWithBigArray.scala | 2 +-
...estLoadDataWithHiveSyntaxDefaultFormat.scala | 2 +-
.../VarcharDataTypesBasicTestCase.scala | 4 +-
.../sql/commands/UsingCarbondataSuite.scala | 2 +-
.../datasource/SparkCarbonDataSourceTest.scala | 4 +-
.../carbondata/CarbonDataSourceSuite.scala | 2 +-
.../carbondata/TestStreamingTableOpName.scala | 19 +-
.../TestStreamingTableWithRowParser.scala | 19 +-
.../processing/loading/model/LoadOption.java | 4 +-
.../sdk/file/CarbonWriterBuilder.java | 4 +-
.../sdk/file/CSVCarbonWriterTest.java | 4 +-
18 files changed, 260 insertions(+), 263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/conf/dataload.properties.template
----------------------------------------------------------------------
diff --git a/conf/dataload.properties.template b/conf/dataload.properties.template
index 0b2f6cf..3b582b2 100644
--- a/conf/dataload.properties.template
+++ b/conf/dataload.properties.template
@@ -51,10 +51,10 @@ delimiter=,
#all_dictionary_path=
#complex column's level 1 delimiter
-#complex_delimiter_level_1=\\$
+#complex_delimiter_level_1='\\\001'
#complex column's level 2 delimiter
-#complex_delimiter_level_2=\\:
+#complex_delimiter_level_2='\\\002'
#timestamp type column's data format
#dateformat=
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/docs/dml-of-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/dml-of-carbondata.md b/docs/dml-of-carbondata.md
index 65654a4..d26cf19 100644
--- a/docs/dml-of-carbondata.md
+++ b/docs/dml-of-carbondata.md
@@ -132,18 +132,18 @@ CarbonData DML statements are documented here,which includes:
- ##### COMPLEX_DELIMITER_LEVEL_1:
- Split the complex type data column in a row (eg., a$b$c --> Array = {a,b,c}).
+ Split the complex type data column in a row (eg., a\001b\001c --> Array = {a,b,c}).
```
- OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='$')
+ OPTIONS('COMPLEX_DELIMITER_LEVEL_1'='\\\001')
```
- ##### COMPLEX_DELIMITER_LEVEL_2:
- Split the complex type nested data column in a row. Applies level_1 delimiter & applies level_2 based on complex data type (eg., a:b$c:d --> Array> = {{a,b},{c,d}}).
+ Split the complex type nested data column in a row. Applies level_1 delimiter & applies level_2 based on complex data type (eg., a\002b\001c\002d --> Array> = {{a,b},{c,d}}).
```
- OPTIONS('COMPLEX_DELIMITER_LEVEL_2'=':')
+ OPTIONS('COMPLEX_DELIMITER_LEVEL_2'='\\\002')
```
- ##### ALL_DICTIONARY_PATH:
@@ -212,8 +212,8 @@ CarbonData DML statements are documented here,which includes:
'FILEHEADER'='empno,empname,designation,doj,workgroupcategory,
workgroupcategoryname,deptno,deptname,projectcode,
projectjoindate,projectenddate,attendance,utilization,salary',
- 'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='$',
- 'COMPLEX_DELIMITER_LEVEL_2'=':',
+ 'MULTILINE'='true','ESCAPECHAR'='\','COMPLEX_DELIMITER_LEVEL_1'='\\\001',
+ 'COMPLEX_DELIMITER_LEVEL_2'='\\\002',
'ALL_DICTIONARY_PATH'='/opt/alldictionary/data.dictionary',
'SINGLE_PASS'='TRUE')
```
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
index 0bcd7e1..dbd2f0e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -338,7 +338,7 @@ public class CarbonTableOutputFormat extends FileOutputFormat<NullWritable, Obje
SKIP_EMPTY_LINE,
carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
- String complexDelim = conf.get(COMPLEX_DELIMITERS, "$" + "," + ":");
+ String complexDelim = conf.get(COMPLEX_DELIMITERS, "\\\001" + "," + "\\\002");
String[] split = complexDelim.split(",");
model.setComplexDelimiterLevel1(split[0]);
if (split.length > 1) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala
index 7fff15d..28edc77 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveComplexType.scala
@@ -46,9 +46,9 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500$abc$20:30:40')")
- sql("insert into adaptive values(2,'600$abc$20:30:40')")
- sql("insert into adaptive values(3,'600$abc$20:30:40')")
+ sql("insert into adaptive values(1,'500\001abc\00120\00230\00240')")
+ sql("insert into adaptive values(2,'600\001abc\00120\00230\00240')")
+ sql("insert into adaptive values(3,'600\001abc\00120\00230\00240')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
Row(2, Row(600, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
@@ -72,9 +72,9 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500$abc$200:300:400')")
- sql("insert into adaptive values(2,'700$abc$200:300:400')")
- sql("insert into adaptive values(3,'800$abc$200:300:400')")
+ sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')")
+ sql("insert into adaptive values(2,'700\001abc\001200\002300\002400')")
+ sql("insert into adaptive values(3,'800\001abc\001200\002300\002400')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
Row(2, Row(700, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
@@ -98,9 +98,9 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
@@ -124,9 +124,9 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
@@ -139,8 +139,8 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:smallint,name:string," +
"marks:array<smallint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'100$abc$20:30:40')")
- sql("insert into adaptive values(2,'200$abc$30:40:50')")
+ sql("insert into adaptive values(1,'100\001abc\00120\00230\00240')")
+ sql("insert into adaptive values(2,'200\001abc\00130\00240\00250')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(100, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
Row(2, Row(200, "abc", mutable.WrappedArray.make(Array(30, 40, 50))))))
@@ -151,8 +151,8 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:smallint,name:string," +
"marks:array<smallint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'500$abc$200:300:400')")
- sql("insert into adaptive values(2,'8000$abc$300:400:500')")
+ sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')")
+ sql("insert into adaptive values(2,'8000\001abc\001300\002400\002500')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
Row(2, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 400, 500))))))
@@ -163,7 +163,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:bigint,name:string," +
"marks:array<bigint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'1$abc$20:30:40')")
+ sql("insert into adaptive values(1,'1\001abc\00120\00230\00240')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(1, "abc", mutable.WrappedArray.make(Array(20, 30, 40))))))
}
@@ -173,8 +173,8 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:bigint,name:string," +
"marks:array<bigint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'500$abc$200:300:400')")
- sql("insert into adaptive values(2,'8000$abc$300:400:500')")
+ sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')")
+ sql("insert into adaptive values(2,'8000\001abc\001300\002400\002500')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
Row(2, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 400, 500))))))
@@ -198,9 +198,9 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:bigint,name:string," +
"marks:array<bigint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
@@ -239,9 +239,9 @@ trait TestAdaptiveComplexType extends QueryTest {
"create table adaptive(roll int, student struct<id:BIGINT,name:string,marks:array<BIGINT>>)" +
" " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
@@ -253,7 +253,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'1.323$abc$2.2:3.3:4.4')")
+ sql("insert into adaptive values(1,'1.323\001abc\0012.2\0023.3\0024.4')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4))))))
sql("Drop table if exists adaptive")
@@ -277,7 +277,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'1.323$abc$20.2:30.3:40.4')")
+ sql("insert into adaptive values(1,'1.323\001abc\00120.2\00230.3\00240.4')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4))))))
sql("Drop table if exists adaptive")
@@ -301,7 +301,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'10.323$abc$20.2:30.3:500.423')")
+ sql("insert into adaptive values(1,'10.323\001abc\00120.2\00230.3\002500.423')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(10.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 500.423))))))
sql("Drop table if exists adaptive")
@@ -325,7 +325,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'1000.323$abc$20.2:30.3:50000.423')")
+ sql("insert into adaptive values(1,'1000.323\001abc\00120.2\00230.3\00250000.423')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(1000.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 50000.423))))))
sql("Drop table if exists adaptive")
@@ -349,7 +349,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'1.797693134862315$abc$2.2:30.3:1.797693134862315')")
+ sql("insert into adaptive values(1,'1.797693134862315\001abc\0012.2\00230.3\0021.797693134862315')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1,
Row(1.797693134862315,
@@ -363,7 +363,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:decimal(3,2),name:string>) stored by " +
"'carbondata'")
- sql("insert into adaptive values(1,'3.2$abc')")
+ sql("insert into adaptive values(1,'3.2\001abc')")
sql("select * from adaptive").show(false)
}
@@ -372,7 +372,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<name:string," +
"marks:array<decimal>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'abc$20.2:30.3:40.4')")
+ sql("insert into adaptive values(1,'abc\00120.2\00230.3\00240.4')")
sql("select * from adaptive").show(false)
}
@@ -383,7 +383,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:timestamp,name:string>) stored by " +
"'carbondata'")
- sql("insert into adaptive values(1,'2017/01/01 00:00:00$abc')")
+ sql("insert into adaptive values(1,'2017/01/01 00:00:00\001abc')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(Timestamp.valueOf("2017-01-01 00:00:00.0"), "abc"))))
}
@@ -395,7 +395,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<name:string," +
"marks:array<timestamp>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'abc$2017/01/01:2018/01/01')")
+ sql("insert into adaptive values(1,'abc\0012017/01/01\0022018/01/01')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1,
Row("abc",
@@ -409,7 +409,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<name:string," +
"marks:array<date>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'abc$2017-01-01')")
+ sql("insert into adaptive values(1,'abc\0012017-01-01')")
sql("select * from adaptive").show(false)
}
@@ -418,7 +418,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'11111$abc$20:30:40')")
+ sql("insert into adaptive values(1,'11111\001abc\00120\00230\00240')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(20, 30, 40))))))
}
@@ -428,7 +428,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'11111$abc$200:300:400')")
+ sql("insert into adaptive values(1,'11111\001abc\001200\002300\002400')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400))))))
sql("Drop table if exists adaptive")
@@ -462,9 +462,9 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
@@ -488,9 +488,9 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
@@ -502,9 +502,9 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000000')")
sql("select * from adaptive").show(false)
}
@@ -513,7 +513,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:short,name:string,marks:array<short>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'11$abc$20:30:40')")
+ sql("insert into adaptive values(1,'11\001abc\00120\00230\00240')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(11, "abc", mutable.WrappedArray.make(Array(20, 30, 40))))))
}
@@ -523,7 +523,7 @@ trait TestAdaptiveComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:SHORT,name:string,marks:array<SHORT>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'11111$abc$200:300:400')")
+ sql("insert into adaptive values(1,'11111\001abc\001200\002300\002400')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400))))))
sql("Drop table if exists adaptive")
@@ -546,7 +546,7 @@ trait TestAdaptiveComplexType extends QueryTest {
"create table adaptive(roll int, student struct<id:boolean,name:string," +
"marks:array<boolean>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'true$abc$false:true:false')")
+ sql("insert into adaptive values(1,'true\001abc\001false\002true\002false')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(true, "abc", mutable.WrappedArray.make(Array(false, true, false))))))
}
@@ -557,12 +557,12 @@ trait TestAdaptiveComplexType extends QueryTest {
"create table adaptive(array1 array<struct<double1:double,double2:double,double3:double>>) " +
"stored by 'carbondata'")
sql(
- "insert into adaptive values('10.35:40000.35:1.7976931348623157$67890985.888:65.5656:200')," +
- "('20.25:50000.25:4.945464565654656546546546324$10000000:300000:3000')")
+ "insert into adaptive values('10.35\00240000.35\0021.7976931348623157\00167890985.888\00265.5656\002200')," +
+ "('20.25\00250000.25\0024.945464565654656546546546324\00110000000\002300000\0023000')")
checkExistence(sql("select * from adaptive"), true, "1.0E7,300000.0,3000.0")
sql("Drop table if exists adaptive")
sql("create table adaptive(struct_arr struct<array_db1:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values('5555555.9559:12345678991234567:3444.999')")
+ sql("insert into adaptive values('5555555.9559\00212345678991234567\0023444.999')")
checkExistence(sql("select * from adaptive"),
true,
"5555555.9559, 1.2345678991234568E16, 3444.999")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala
index 528fb69..d0b1df8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingForNullValues.scala
@@ -53,7 +53,7 @@ class TestAdaptiveEncodingForNullValues
sql(
"create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'null$abc$null:null:null')")
+ sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -64,7 +64,7 @@ class TestAdaptiveEncodingForNullValues
sql(
"create table adaptive(roll int, student struct<id:smallint,name:string," +
"marks:array<smallint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'null$abc$null:null:null')")
+ sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -75,7 +75,7 @@ class TestAdaptiveEncodingForNullValues
sql(
"create table adaptive(roll int, student struct<id:bigint,name:string," +
"marks:array<bigint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'null$abc$null:null:null')")
+ sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -85,7 +85,7 @@ class TestAdaptiveEncodingForNullValues
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'null$abc$null:null:null')")
+ sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -96,7 +96,7 @@ class TestAdaptiveEncodingForNullValues
"create table adaptive(roll int, student struct<id:decimal(3,2),name:string," +
"marks:array<decimal>>) stored by " +
"'carbondata'")
- sql("insert into adaptive values(1,'null$abc$null:null:null')")
+ sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -108,7 +108,7 @@ class TestAdaptiveEncodingForNullValues
sql(
"create table adaptive(roll int, student struct<id:timestamp,name:string>) stored by " +
"'carbondata'")
- sql("insert into adaptive values(1,'null$abc')")
+ sql("insert into adaptive values(1,'null\001abc')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(null, "abc"))))
}
@@ -120,7 +120,7 @@ class TestAdaptiveEncodingForNullValues
sql(
"create table adaptive(roll int, student struct<name:string," +
"marks:array<timestamp>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'abc$null:null:null')")
+ sql("insert into adaptive values(1,'abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row("abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -130,7 +130,7 @@ class TestAdaptiveEncodingForNullValues
sql(
"create table adaptive(roll int, student struct<name:string," +
"marks:array<date>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'abc$null:null:null')")
+ sql("insert into adaptive values(1,'abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row("abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -140,7 +140,7 @@ class TestAdaptiveEncodingForNullValues
sql(
"create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'null$abc$null:null:null')")
+ sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -150,7 +150,7 @@ class TestAdaptiveEncodingForNullValues
sql(
"create table adaptive(roll int, student struct<id:short,name:string,marks:array<short>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'null$abc$null:null:null')")
+ sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
@@ -161,7 +161,7 @@ class TestAdaptiveEncodingForNullValues
"create table adaptive(roll int, student struct<id:boolean,name:string," +
"marks:array<boolean>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'null$abc$null:null:null')")
+ sql("insert into adaptive values(1,'null\001abc\001null\002null\002null')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(null, "abc", mutable.WrappedArray.make(Array(null, null, null))))))
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala
index 6ff8a15..a353ec0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCompactionComplexType.scala
@@ -46,10 +46,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500$abc$20:30:40')")
- sql("insert into adaptive values(2,'600$abc$30:30:40')")
- sql("insert into adaptive values(3,'700$abc$40:30:40')")
- sql("insert into adaptive values(4,'800$abc$50:30:40')")
+ sql("insert into adaptive values(1,'500\001abc\00120\00230\00240')")
+ sql("insert into adaptive values(2,'600\001abc\00130\00230\00240')")
+ sql("insert into adaptive values(3,'700\001abc\00140\00230\00240')")
+ sql("insert into adaptive values(4,'800\001abc\00150\00230\00240')")
sql("alter table adaptive compact 'major'").show(200,false)
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
@@ -75,10 +75,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500$abc$200:300:400')")
- sql("insert into adaptive values(2,'600$abc$300:300:400')")
- sql("insert into adaptive values(3,'700$abc$400:300:400')")
- sql("insert into adaptive values(4,'800$abc$500:300:400')")
+ sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')")
+ sql("insert into adaptive values(2,'600\001abc\001300\002300\002400')")
+ sql("insert into adaptive values(3,'700\001abc\001400\002300\002400')")
+ sql("insert into adaptive values(4,'800\001abc\001500\002300\002400')")
sql("alter table adaptive compact 'major'").show(200,false)
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
@@ -104,10 +104,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(2,'70000$abc$2000000:4000000:4000000')")
- sql("insert into adaptive values(3,'100000$abc$2000000:5000000:4000000')")
- sql("insert into adaptive values(4,'200000$abc$2000000:6000000:4000000')")
+ sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(2,'70000\001abc\0012000000\0024000000\0024000000')")
+ sql("insert into adaptive values(3,'100000\001abc\0012000000\0025000000\0024000000')")
+ sql("insert into adaptive values(4,'200000\001abc\0012000000\0026000000\0024000000')")
sql("alter table adaptive compact 'major'").show(200,false)
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
@@ -133,10 +133,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:int,name:string,marks:array<int>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
- sql("insert into adaptive values(2,'700000$abc$210:350:52000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
- sql("insert into adaptive values(4,'10000001$abd$250:450:62000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001210\002350\00252000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(4,'10000001\001abd\001250\002450\00262000000')")
sql("alter table adaptive compact 'major'").show(200,false)
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
@@ -151,10 +151,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:smallint,name:string," +
"marks:array<smallint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'100$abc$20:30:40')")
- sql("insert into adaptive values(2,'200$abc$30:40:50')")
- sql("insert into adaptive values(3,'300$abd$30:41:55')")
- sql("insert into adaptive values(4,'400$abe$30:42:56')")
+ sql("insert into adaptive values(1,'100\001abc\00120\00230\00240')")
+ sql("insert into adaptive values(2,'200\001abc\00130\00240\00250')")
+ sql("insert into adaptive values(3,'300\001abd\00130\00241\00255')")
+ sql("insert into adaptive values(4,'400\001abe\00130\00242\00256')")
sql("alter table adaptive compact 'major'").show(200,false)
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(100, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
@@ -168,20 +168,20 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:smallint,name:string," +
"marks:array<smallint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'500$abc$200:300:400')")
- sql("insert into adaptive values(2,'8000$abc$300:410:500')")
- sql("insert into adaptive values(3,'9000$abee$310:420:400')")
- sql("insert into adaptive values(4,'9900$abfffffffffffffff$320:430:500')")
+ sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')")
+ sql("insert into adaptive values(2,'8000\001abc\001300\002410\002500')")
+ sql("insert into adaptive values(3,'9000\001abee\001310\002420\002400')")
+ sql("insert into adaptive values(4,'9900\001abfffffffffffffff\001320\002430\002500')")
sql("alter table adaptive compact 'major'").show(200,false)
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
Row(2, Row(8000, "abc", mutable.WrappedArray.make(Array(300, 410, 500)))),
Row(3, Row(9000, "abee", mutable.WrappedArray.make(Array(310, 420, 400)))),
Row(4, Row(9900, "abfffffffffffffff", mutable.WrappedArray.make(Array(320, 430, 500))))))
- sql("insert into adaptive values(5,'500$abc$200:310:400')")
- sql("insert into adaptive values(6,'8000$abc$300:310:500')")
- sql("insert into adaptive values(7,'9000$abee$310:320:400')")
- sql("insert into adaptive values(8,'9900$abfffffffffffffffeeee$320:330:500')")
+ sql("insert into adaptive values(5,'500\001abc\001200\002310\002400')")
+ sql("insert into adaptive values(6,'8000\001abc\001300\002310\002500')")
+ sql("insert into adaptive values(7,'9000\001abee\001310\002320\002400')")
+ sql("insert into adaptive values(8,'9900\001abfffffffffffffffeeee\001320\002330\002500')")
sql("alter table adaptive compact 'major'").show(200,false)
sql("SHOW SEGMENTS FOR TABLE adaptive").show(200,false)
sql("clean files for table adaptive").show(200,false)
@@ -202,18 +202,18 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:bigint,name:string," +
"marks:array<bigint>>) stored by 'carbondata'")
- sql("insert into adaptive values(11,'1$abc$21:30:40')")
- sql("insert into adaptive values(12,'1$ab1$22:30:40')")
- sql("insert into adaptive values(13,'1$ab2$23:30:40')")
- sql("insert into adaptive values(14,'1$ab3$24:30:40')")
- sql("insert into adaptive values(15,'1$ab4$25:30:40')")
- sql("insert into adaptive values(16,'1$ab5$26:30:40')")
- sql("insert into adaptive values(17,'1$ab6$27:30:40')")
- sql("insert into adaptive values(18,'1$ab7$28:30:40')")
- sql("insert into adaptive values(19,'1$ab8$29:30:40')")
- sql("insert into adaptive values(20,'1$ab9$30:30:40')")
- sql("insert into adaptive values(21,'1$ab10$31:30:40')")
- sql("insert into adaptive values(22,'1$ab11$32:30:40')")
+ sql("insert into adaptive values(11,'1\001abc\00121\00230\00240')")
+ sql("insert into adaptive values(12,'1\001ab1\00122\00230\00240')")
+ sql("insert into adaptive values(13,'1\001ab2\00123\00230\00240')")
+ sql("insert into adaptive values(14,'1\001ab3\00124\00230\00240')")
+ sql("insert into adaptive values(15,'1\001ab4\00125\00230\00240')")
+ sql("insert into adaptive values(16,'1\001ab5\00126\00230\00240')")
+ sql("insert into adaptive values(17,'1\001ab6\00127\00230\00240')")
+ sql("insert into adaptive values(18,'1\001ab7\00128\00230\00240')")
+ sql("insert into adaptive values(19,'1\001ab8\00129\00230\00240')")
+ sql("insert into adaptive values(20,'1\001ab9\00130\00230\00240')")
+ sql("insert into adaptive values(21,'1\001ab10\00131\00230\00240')")
+ sql("insert into adaptive values(22,'1\001ab11\00132\00230\00240')")
sql("alter table adaptive compact 'major'").show(200,false)
sql("SHOW SEGMENTS FOR TABLE adaptive").show(200,false)
sql("clean files for table adaptive").show(200,false)
@@ -240,10 +240,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:bigint,name:string," +
"marks:array<bigint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'500$abc$200:300:400')")
- sql("insert into adaptive values(2,'8000$abc$300:400:500')")
- sql("insert into adaptive values(3,'9000$abc$300:400:500')")
- sql("insert into adaptive values(4,'10000$abc$300:400:500')")
+ sql("insert into adaptive values(1,'500\001abc\001200\002300\002400')")
+ sql("insert into adaptive values(2,'8000\001abc\001300\002400\002500')")
+ sql("insert into adaptive values(3,'9000\001abc\001300\002400\002500')")
+ sql("insert into adaptive values(4,'10000\001abc\001300\002400\002500')")
sql("alter table adaptive compact'major'")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
@@ -293,18 +293,18 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:bigint,name:string," +
"marks:array<bigint>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')")
sql("alter table adaptive compact'major'")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
@@ -400,18 +400,18 @@ class TestCompactionComplexType extends QueryTest {
"create table adaptive(roll int, student struct<id:BIGINT,name:string,marks:array<BIGINT>>)" +
" " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
@@ -434,10 +434,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'1.323$abc$2.2:3.3:4.4')")
- sql("insert into adaptive values(2,'1.324$abc$2.2:3.3:4.4')")
- sql("insert into adaptive values(3,'1.325$abc$2.2:3.3:4.4')")
- sql("insert into adaptive values(4,'1.326$abc$2.2:3.3:4.4')")
+ sql("insert into adaptive values(1,'1.323\001abc\0012.2\0023.3\0024.4')")
+ sql("insert into adaptive values(2,'1.324\001abc\0012.2\0023.3\0024.4')")
+ sql("insert into adaptive values(3,'1.325\001abc\0012.2\0023.3\0024.4')")
+ sql("insert into adaptive values(4,'1.326\001abc\0012.2\0023.3\0024.4')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(2.2, 3.3, 4.4)))),
@@ -491,10 +491,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'1.323$abc$20.2:30.3:40.4')")
- sql("insert into adaptive values(2,'1.324$abc$20.2:30.3:40.5')")
- sql("insert into adaptive values(3,'1.325$abc$20.2:30.3:40.6')")
- sql("insert into adaptive values(4,'1.326$abc$20.2:30.3:40.7')")
+ sql("insert into adaptive values(1,'1.323\001abc\00120.2\00230.3\00240.4')")
+ sql("insert into adaptive values(2,'1.324\001abc\00120.2\00230.3\00240.5')")
+ sql("insert into adaptive values(3,'1.325\001abc\00120.2\00230.3\00240.6')")
+ sql("insert into adaptive values(4,'1.326\001abc\00120.2\00230.3\00240.7')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(1.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 40.4)))),
@@ -549,10 +549,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'10.323$abc$20.2:30.3:501.423')")
- sql("insert into adaptive values(2,'10.323$abc$20.2:30.3:502.421')")
- sql("insert into adaptive values(3,'10.323$abc$20.2:30.3:503.422')")
- sql("insert into adaptive values(4,'10.323$abc$20.2:30.3:504.424')")
+ sql("insert into adaptive values(1,'10.323\001abc\00120.2\00230.3\002501.423')")
+ sql("insert into adaptive values(2,'10.323\001abc\00120.2\00230.3\002502.421')")
+ sql("insert into adaptive values(3,'10.323\001abc\00120.2\00230.3\002503.422')")
+ sql("insert into adaptive values(4,'10.323\001abc\00120.2\00230.3\002504.424')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(10.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 501.423)))),
@@ -606,10 +606,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'1000.323$abc$20.2:30.3:60000.423')")
- sql("insert into adaptive values(2,'1000.324$abc$20.2:30.3:70000.424')")
- sql("insert into adaptive values(3,'1000.325$abc$20.2:30.3:80000.425')")
- sql("insert into adaptive values(4,'1000.326$abc$20.2:30.3:90000.426')")
+ sql("insert into adaptive values(1,'1000.323\001abc\00120.2\00230.3\00260000.423')")
+ sql("insert into adaptive values(2,'1000.324\001abc\00120.2\00230.3\00270000.424')")
+ sql("insert into adaptive values(3,'1000.325\001abc\00120.2\00230.3\00280000.425')")
+ sql("insert into adaptive values(4,'1000.326\001abc\00120.2\00230.3\00290000.426')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(1000.323, "abc", mutable.WrappedArray.make(Array(20.2, 30.3, 60000.423)))),
@@ -664,10 +664,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:double,name:string," +
"marks:array<double>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'1.797693134862315$abc$2.2:30.3:1.797693134862315')")
- sql("insert into adaptive values(2,'1.797693134862316$abc$2.2:30.3:1.797693134862316')")
- sql("insert into adaptive values(3,'1.797693134862317$abc$2.2:30.3:1.797693134862317')")
- sql("insert into adaptive values(4,'1.797693134862318$abc$2.2:30.3:1.797693134862318')")
+ sql("insert into adaptive values(1,'1.797693134862315\001abc\0012.2\00230.3\0021.797693134862315')")
+ sql("insert into adaptive values(2,'1.797693134862316\001abc\0012.2\00230.3\0021.797693134862316')")
+ sql("insert into adaptive values(3,'1.797693134862317\001abc\0012.2\00230.3\0021.797693134862317')")
+ sql("insert into adaptive values(4,'1.797693134862318\001abc\0012.2\00230.3\0021.797693134862318')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1,
@@ -695,7 +695,7 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:decimal(3,2),name:string>) stored by " +
"'carbondata'")
- sql("insert into adaptive values(1,'3.2$abc')")
+ sql("insert into adaptive values(1,'3.2\001abc')")
sql("select * from adaptive").show(false)
}
@@ -704,7 +704,7 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<name:string," +
"marks:array<decimal>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'abc$20.2:30.3:40.4')")
+ sql("insert into adaptive values(1,'abc\00120.2\00230.3\00240.4')")
sql("select * from adaptive").show(false)
}
@@ -715,10 +715,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:timestamp,name:string>) stored by " +
"'carbondata'")
- sql("insert into adaptive values(1,'2017/01/01 00:00:00$abc')")
- sql("insert into adaptive values(2,'2017/01/02 00:00:00$abc')")
- sql("insert into adaptive values(3,'2017/01/03 00:00:00$abc')")
- sql("insert into adaptive values(4,'2017/01/04 00:00:00$abc')")
+ sql("insert into adaptive values(1,'2017/01/01 00:00:00\001abc')")
+ sql("insert into adaptive values(2,'2017/01/02 00:00:00\001abc')")
+ sql("insert into adaptive values(3,'2017/01/03 00:00:00\001abc')")
+ sql("insert into adaptive values(4,'2017/01/04 00:00:00\001abc')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(Timestamp.valueOf("2017-01-01 00:00:00.0"), "abc")),
@@ -735,10 +735,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<name:string," +
"marks:array<timestamp>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'abc1$2017/01/01:2018/01/01')")
- sql("insert into adaptive values(2,'abc2$2017/01/02:2018/01/03')")
- sql("insert into adaptive values(3,'abc3$2017/01/04:2018/01/05')")
- sql("insert into adaptive values(4,'abc4$2017/01/06:2018/01/07')")
+ sql("insert into adaptive values(1,'abc1\0012017/01/01\0022018/01/01')")
+ sql("insert into adaptive values(2,'abc2\0012017/01/02\0022018/01/03')")
+ sql("insert into adaptive values(3,'abc3\0012017/01/04\0022018/01/05')")
+ sql("insert into adaptive values(4,'abc4\0012017/01/06\0022018/01/07')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1,
@@ -769,7 +769,7 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<name:string," +
"marks:array<date>>) stored by 'carbondata'")
- sql("insert into adaptive values(1,'abc$2017-01-01')")
+ sql("insert into adaptive values(1,'abc\0012017-01-01')")
sql("select * from adaptive").show(false)
}
@@ -778,10 +778,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'11111$abc$20:30:40')")
- sql("insert into adaptive values(2,'11111$abc$55:65:75')")
- sql("insert into adaptive values(3,'11111$abc$88:98:8')")
- sql("insert into adaptive values(4,'11111$abc$99:9:19')")
+ sql("insert into adaptive values(1,'11111\001abc\00120\00230\00240')")
+ sql("insert into adaptive values(2,'11111\001abc\00155\00265\00275')")
+ sql("insert into adaptive values(3,'11111\001abc\00188\00298\0028')")
+ sql("insert into adaptive values(4,'11111\001abc\00199\0029\00219')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(20, 30, 40)))),
@@ -796,10 +796,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:long,name:string,marks:array<long>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'11111$abc$200:300:400')")
- sql("insert into adaptive values(2,'11111$abc$201:301:401')")
- sql("insert into adaptive values(3,'11111$abc$202:302:402')")
- sql("insert into adaptive values(4,'11111$abc$203:303:403')")
+ sql("insert into adaptive values(1,'11111\001abc\001200\002300\002400')")
+ sql("insert into adaptive values(2,'11111\001abc\001201\002301\002401')")
+ sql("insert into adaptive values(3,'11111\001abc\001202\002302\002402')")
+ sql("insert into adaptive values(4,'11111\001abc\001203\002303\002403')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
@@ -861,9 +861,9 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'50000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(2,'70000$abc$2000000:3000000:4000000')")
- sql("insert into adaptive values(3,'100000$abc$2000000:3000000:4000000')")
+ sql("insert into adaptive values(1,'50000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(2,'70000\001abc\0012000000\0023000000\0024000000')")
+ sql("insert into adaptive values(3,'100000\001abc\0012000000\0023000000\0024000000')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(50000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
Row(2, Row(70000, "abc", mutable.WrappedArray.make(Array(2000000, 3000000, 4000000)))),
@@ -887,9 +887,9 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(500000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
Row(2, Row(700000, "abc", mutable.WrappedArray.make(Array(200, 300, 52000000)))),
@@ -901,9 +901,9 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:LONG,name:string,marks:array<LONG>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'500000$abc$200:300:52000000000')")
- sql("insert into adaptive values(2,'700000$abc$200:300:52000000000')")
- sql("insert into adaptive values(3,'10000000$abc$200:300:52000000000')")
+ sql("insert into adaptive values(1,'500000\001abc\001200\002300\00252000000000')")
+ sql("insert into adaptive values(2,'700000\001abc\001200\002300\00252000000000')")
+ sql("insert into adaptive values(3,'10000000\001abc\001200\002300\00252000000000')")
sql("select * from adaptive").show(false)
}
@@ -912,7 +912,7 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:short,name:string,marks:array<short>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'11$abc$20:30:40')")
+ sql("insert into adaptive values(1,'11\001abc\00120\00230\00240')")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(11, "abc", mutable.WrappedArray.make(Array(20, 30, 40))))))
}
@@ -922,10 +922,10 @@ class TestCompactionComplexType extends QueryTest {
sql(
"create table adaptive(roll int, student struct<id:SHORT,name:string,marks:array<SHORT>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'11111$abc$200:300:400')")
- sql("insert into adaptive values(1,'11111$abc$200:300:401')")
- sql("insert into adaptive values(1,'11111$abc$200:300:402')")
- sql("insert into adaptive values(1,'11111$abc$200:300:403')")
+ sql("insert into adaptive values(1,'11111\001abc\001200\002300\002400')")
+ sql("insert into adaptive values(1,'11111\001abc\001200\002300\002401')")
+ sql("insert into adaptive values(1,'11111\001abc\001200\002300\002402')")
+ sql("insert into adaptive values(1,'11111\001abc\001200\002300\002403')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(11111, "abc", mutable.WrappedArray.make(Array(200, 300, 400)))),
@@ -976,10 +976,10 @@ class TestCompactionComplexType extends QueryTest {
"create table adaptive(roll int, student struct<id:boolean,name:string," +
"marks:array<boolean>>) " +
"stored by 'carbondata'")
- sql("insert into adaptive values(1,'true$abc$false:true:false')")
- sql("insert into adaptive values(1,'true$abc$false:true:true')")
- sql("insert into adaptive values(1,'true$abc$false:true:true')")
- sql("insert into adaptive values(1,'true$abc$false:true:false')")
+ sql("insert into adaptive values(1,'true\001abc\001false\002true\002false')")
+ sql("insert into adaptive values(1,'true\001abc\001false\002true\002true')")
+ sql("insert into adaptive values(1,'true\001abc\001false\002true\002true')")
+ sql("insert into adaptive values(1,'true\001abc\001false\002true\002false')")
sql("alter table adaptive compact 'major' ")
checkAnswer(sql("select * from adaptive"),
Seq(Row(1, Row(true, "abc", mutable.WrappedArray.make(Array(false, true, false)))),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
index f2e33f3..40ff648 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
@@ -58,7 +58,8 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (roll string,person array<int>) stored by " +
"'carbondata'")
- sql("insert into table1 values('abc','1$2$3')")
+ sql("insert into table1 values('abc','1\0012\0013')")
+ sql("select * from table1").show(false)
checkAnswer(sql("select roll,person from table1"),
Seq(Row("abc", mutable.WrappedArray.make(Array(1, 2, 3)))))
}
@@ -68,7 +69,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (roll int,person Struct<detail:array<int>>) stored by " +
"'carbondata'")
- sql("insert into table1 values(1,'1:2')")
+ sql("insert into table1 values(1,'1\0022')")
checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(1)))
checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(2)))
checkAnswer(sql("select roll,person from table1"),
@@ -96,7 +97,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (roll int,person Struct<detail:array<string>>) stored by " +
"'carbondata'")
- sql("insert into table1 values(1,'abc:bcd')")
+ sql("insert into table1 values(1,'abc\002bcd')")
checkAnswer(sql("select person.detail[0] from table1"), Seq(Row("abc")))
checkAnswer(sql("select person.detail[1] from table1"), Seq(Row("bcd")))
checkAnswer(sql("select roll,person from table1"),
@@ -122,7 +123,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (roll int,person Struct<detail:array<double>>) stored by " +
"'carbondata'")
- sql("insert into table1 values(1,'10.00:20.00')")
+ sql("insert into table1 values(1,'10.00\00220.00')")
checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(10.0)))
checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(20.0)))
checkAnswer(sql("select roll,person from table1"),
@@ -145,7 +146,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (roll int,person Struct<detail:array<decimal(3,2)>>) stored by " +
"'carbondata'")
- sql("insert into table1 values(1,'3.4:4.2')")
+ sql("insert into table1 values(1,'3.4\0024.2')")
checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(3.40)))
checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(4.20)))
checkAnswer(sql("select roll,person.detail[0] from table1"), Seq(Row(1, 3.40)))
@@ -178,7 +179,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (roll int,person Struct<detail:array<timestamp>>) stored by " +
"'carbondata'")
- sql("insert into table1 select 1,'2018/01/01:2017/01/01'")
+ sql("insert into table1 select 1,'2018/01/01\0022017/01/01'")
checkExistence(sql("select person.detail[0] from table1"), true, "2018-01-01 00:00:00.0")
checkExistence(sql("select person.detail[1] from table1"), true, "2017-01-01 00:00:00.0")
checkAnswer(sql("select roll,person from table1"),
@@ -208,7 +209,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (roll int,person Struct<detail:array<long>>) stored by " +
"'carbondata'")
- sql("insert into table1 values(1,'2018888:2018889')")
+ sql("insert into table1 values(1,'2018888\0022018889')")
checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(2018888)))
checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(2018889)))
checkAnswer(sql("select person,roll from table1"),
@@ -232,7 +233,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (roll int,person Struct<detail:array<short>>) stored by " +
"'carbondata'")
- sql("insert into table1 values(1,'20:30')")
+ sql("insert into table1 values(1,'20\00230')")
checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(20)))
checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(30)))
checkAnswer(sql("select person,roll from table1"),
@@ -256,7 +257,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (roll int,person Struct<detail:array<boolean>>) stored by " +
"'carbondata'")
- sql("insert into table1 values(1,'true:false')")
+ sql("insert into table1 values(1,'true\002false')")
checkAnswer(sql("select person.detail[0] from table1"), Seq(Row(true)))
checkAnswer(sql("select person.detail[1] from table1"), Seq(Row(false)))
checkAnswer(sql("select person,roll from table1"),
@@ -374,7 +375,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table table1 (person Struct<detail:string,ph:array<int>>) stored by " +
"'carbondata' tblproperties('dictionary_include'='person')")
- sql("insert into table1 values ('abc$2')")
+ sql("insert into table1 values ('abc\0012')")
sql("select person from table1").show(false)
sql("select person.detail, person.ph[0] from table1").show(false)
}
@@ -518,14 +519,14 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
"create table table1 (roll string,person Struct<detail:int,age:string>,person1 " +
"Struct<detail:int,age:array<string>>) stored by " +
"'carbondata'")
- sql("insert into table1 values('abc','1$abc','2$cde')")
+ sql("insert into table1 values('abc','1\001abc','2\001cde')")
sql("select person.detail,person1.age from table1").show(false)
}
test("test Projection PushDown for more than one Struct column Cases -1") {
sql("drop table if exists test")
sql("create table test (a struct<b:int, c:struct<d:int,e:int>>) stored by 'carbondata'")
- sql("insert into test select '1$2:3'")
+ sql("insert into test select '1\0012\0023'")
checkAnswer(sql("select * from test"), Seq(Row(Row(1, Row(2, 3)))))
checkAnswer(sql("select a.b,a.c from test"), Seq(Row(1, Row(2, 3))))
checkAnswer(sql("select a.c, a.b from test"), Seq(Row(Row(2, 3), 1)))
@@ -552,9 +553,9 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
"h:string,i:int>,j:int>) stored " +
"by " +
"'carbondata'")
- sql("insert into table1 values(1,'1$abc$2$efg$3:mno:4$5')")
- sql("insert into table1 values(2,'1$abc$2$efg$3:mno:4$5')")
- sql("insert into table1 values(3,'1$abc$2$efg$3:mno:4$5')")
+ sql("insert into table1 values(1,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
+ sql("insert into table1 values(2,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
+ sql("insert into table1 values(3,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
checkAnswer(sql("select a.b from table1"), Seq(Row(1), Row(1), Row(1)))
checkAnswer(sql("select a.c from table1"), Seq(Row("abc"), Row("abc"), Row("abc")))
checkAnswer(sql("select a.d from table1"), Seq(Row(2), Row(2), Row(2)))
@@ -596,9 +597,9 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
"h:string,i:int>,j:int>) stored " +
"by " +
"'carbondata' tblproperties('dictionary_include'='a')")
- sql("insert into table1 values(1,'1$abc$2$efg$3:mno:4$5')")
- sql("insert into table1 values(2,'1$abc$2$efg$3:mno:4$5')")
- sql("insert into table1 values(3,'1$abc$2$efg$3:mno:4$5')")
+ sql("insert into table1 values(1,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
+ sql("insert into table1 values(2,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
+ sql("insert into table1 values(3,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
checkAnswer(sql("select a.b from table1"), Seq(Row(1), Row(1), Row(1)))
checkAnswer(sql("select a.c from table1"), Seq(Row("abc"), Row("abc"), Row("abc")))
@@ -656,7 +657,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
"create table test(cus_id string, struct_of_array struct<id:int,date:timestamp," +
"sno:array<int>,sal:array<double>,state:array<string>,date1:array<timestamp>>) stored by " +
"'carbondata'")
- sql("insert into test values('cus_01','1$2017/01/01$1:2$2.0:3.0$ab:ac$2018/01/01')")
+ sql("insert into test values('cus_01','1\0012017/01/01\0011\0022\0012.0\0023.0\001ab\002ac\0012018/01/01')")
// sql("select *from test").show(false)
sql(
"select struct_of_array.state[0],count(distinct struct_of_array.id) as count_int,count" +
@@ -672,7 +673,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS test")
sql("create table test(cus_id string,array_of_struct array<struct<id:int,country:string," +
"state:string,city:string>>) stored by 'carbondata'")
- sql("insert into test values('cus_01','123:abc:mno:xyz$1234:abc1:mno1:xyz1')")
+ sql("insert into test values('cus_01','123\002abc\002mno\002xyz\0011234\002abc1\002mno1\002xyz1')")
checkAnswer(sql("select array_of_struct.state[0],count(distinct array_of_struct.id[0]) as count_country," +
"count(distinct array_of_struct.state[0]) as count_city from test group by array_of_struct" +
".state[0]"), Seq(Row("mno", 1, 1)))
@@ -681,9 +682,9 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
test("test struct complex type with filter") {
sql("DROP TABLE IF EXISTS test")
sql("create table test(id int,a struct<b:int,c:int>) stored by 'carbondata'")
- sql("insert into test values(1,'2$3')")
- sql("insert into test values(3,'5$3')")
- sql("insert into test values(2,'4$5')")
+ sql("insert into test values(1,'2\0013')")
+ sql("insert into test values(3,'5\0013')")
+ sql("insert into test values(2,'4\0015')")
checkAnswer(sql("select a.b from test where id=3"),Seq(Row(5)))
checkAnswer(sql("select a.b from test where a.c!=3"),Seq(Row(4)))
checkAnswer(sql("select a.b from test where a.c=3"),Seq(Row(5),Row(2)))
@@ -710,7 +711,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
test("test Projection with two struct") {
sql("DROP TABLE IF EXISTS test")
sql("create table test(id int,a struct<b:int,c:int>, d struct<e:int,f:int>) stored by 'carbondata'")
- sql("insert into test values(1,'2$3','3$2')")
+ sql("insert into test values(1,'2\0013','3\0012')")
checkAnswer(sql("select * from test"),Seq(Row(1,Row(2,3),Row(3,2))))
checkAnswer(sql("select a.b,id,a.c from test"),Seq(Row(2,1,3)))
checkAnswer(sql("select d.e,d.f from test"),Seq(Row(3,2)))
@@ -730,7 +731,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
test("test project with struct and array") {
sql("DROP TABLE IF EXISTS test")
sql("create table test(id int,a struct<b:int,c:int>, d struct<e:int,f:int>,person Struct<detail:array<int>>) stored by 'carbondata'")
- sql("insert into test values(1,'2$3','3$2','5:6:7:8')")
+ sql("insert into test values(1,'2\0013','3\0012','5\0026\0027\0028')")
checkAnswer(sql("select * from test"),Seq(Row(1,Row(2,3),Row(3,2),Row(mutable.WrappedArray.make(Array(5,6,7,8))))))
checkAnswer(sql("select a.b,id,a.c,person.detail[0] from test"),Seq(Row(2,1,3,5)))
checkAnswer(sql("select a.b,id,a.c,person.detail[0],d.e,d.f,person.detail[1],id from test"),Seq(Row(2,1,3,5,3,2,6,1)))
@@ -740,7 +741,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
test("test block Update for complex datatype") {
sql("DROP TABLE IF EXISTS test")
sql("create table test(id int,a struct<b:int,c:int>,d array<int>) stored by 'carbondata'")
- sql("insert into test values(1,'2$3',4)")
+ sql("insert into test values(1,'2\0013',4)")
val structException = intercept[UnsupportedOperationException](
sql("update test set(a.b)=(4) where id=1").show(false))
assertResult("Unsupported operation on Complex data type")(structException.getMessage)
@@ -809,7 +810,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
"h:string,i:int>,j:int>) stored " +
"by " +
"'carbondata' tblproperties('dictionary_exclude'='a')")
- sql("insert into table1 values(1,'1$abc$2$efg$3:mno:4$5')")
+ sql("insert into table1 values(1,'1\001abc\0012\001efg\0013\002mno\0024\0015')")
checkAnswer(sql("select a.b from table1"), Seq(Row(1)))
sql("DROP TABLE IF EXISTS table1")
val structException = intercept[MalformedCarbonCommandException](
@@ -872,7 +873,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS table1")
sql(
"create table table1 (person struct<b:array<double>>) stored by 'carbondata'")
- sql("insert into table1 values('10000000:2000000000:2900000000')")
+ sql("insert into table1 values('10000000\0022000000000\0022900000000')")
checkExistence(sql("select * from table1"),true,"2.9E9")
}
@@ -924,7 +925,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql(
"create table test(id int,a struct<b:int,c:int>, d struct<e:int,f:int>, d1 struct<e1:int," +
"f1:int>) stored by 'carbondata' tblproperties('dictionary_include'='d1')")
- sql("insert into test values(1,'2$3','4$5','6$7')")
+ sql("insert into test values(1,'2\0013','4\0015','6\0017')")
checkAnswer(sql("select * from test"),Seq(Row(1,Row(2,3),Row(4,5),Row(6,7))))
sql("DROP TABLE IF EXISTS test")
sql(
@@ -971,7 +972,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
"MM-dd-yyyy")
sql("DROP TABLE IF EXISTS test")
sql("create table test(a struct<d1:date,d2:date>) stored by 'carbondata'")
- sql("insert into test values ('02-18-2012$12-9-2016')")
+ sql("insert into test values ('02-18-2012\00112-9-2016')")
checkAnswer(sql("select * from test "), Row(Row(java.sql.Date.valueOf("2012-02-18"),java.sql.Date.valueOf("2016-12-09"))))
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT,
@@ -981,7 +982,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS table1")
sql(
"create table table1 (id int, name string, structField struct<intval:int, stringval:string>) stored by 'carbondata'")
- sql("insert into table1 values(null,'aaa','23$bb')")
+ sql("insert into table1 values(null,'aaa','23\001bb')")
checkAnswer(sql("select * from table1"),Seq(Row(null,"aaa", Row(23,"bb"))))
checkAnswer(sql("select id,name,structField.intval,structField.stringval from table1"),Seq(Row(null,"aaa",23,"bb")))
checkAnswer(sql("select id,name,structField.intval,structField.stringval,name from table1"),Seq(Row(null,"aaa",23,"bb","aaa")))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala
index f4fd168..ac793ab 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeWithBigArray.scala
@@ -33,7 +33,7 @@ class TestComplexTypeWithBigArray extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
// write a CSV containing 32000 row, each row has an array with 10 elements
val out = new PrintStream(new FileOutputStream(file))
- (1 to 33000).foreach(i=>out.println(s"$i,$i$$1"))
+ (1 to 33000).foreach(i=>out.println(s"$i,$i\0011"))
out.close()
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala
index 7f150be..61271e1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala
@@ -373,7 +373,7 @@ class TestLoadDataWithHiveSyntaxDefaultFormat extends QueryTest with BeforeAndAf
"('dictionary_include'='date1,date2')")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
- sql("insert into array_timestamp values('2015/01/01$2016/01/01','2017/01/01')")
+ sql("insert into array_timestamp values('2015/01/01\0012016/01/01','2017/01/01')")
checkExistence(sql("select * from array_timestamp "),
true, "2015-01-01 00:00:00.0, 2016-01-01 00:00:00.0")
checkExistence(sql("select * from array_timestamp "),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
index 4051de4..a96f7df 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -331,8 +331,8 @@ class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach wi
sql(
"""
| INSERT INTO TABLE varchar_complex_table
- | VALUES(1,'ar1.0$ar1.1','longstr10','normal string1','longstr11','ar2.0$ar2.1'),
- | (2,'ar1.2$ar1.3','longstr20','normal string2','longstr21','ar2.2$ar2.3')
+ | VALUES(1,'ar1.0\001ar1.1','longstr10','normal string1','longstr11','ar2.0\001ar2.1'),
+ | (2,'ar1.2\001ar1.3','longstr20','normal string2','longstr21','ar2.2\001ar2.3')
| """.stripMargin)
checkAnswer(
sql("SELECT * FROM varchar_complex_table where varchar1='longstr10'"),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala
index 74e04b0..eafbf36 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/UsingCarbondataSuite.scala
@@ -44,7 +44,7 @@ class UsingCarbondataSuite extends QueryTest with BeforeAndAfterEach {
sql("DROP TABLE IF EXISTS create_source")
sql("CREATE TABLE create_source(intField INT, stringField STRING, complexField ARRAY<INT>) " +
"USING carbondata")
- sql("""INSERT INTO create_source VALUES(1,"source","1$2$3")""")
+ sql("""INSERT INTO create_source VALUES(1,"source","1\0012\0013")""")
checkAnswer(sql("SELECT * FROM create_source"), Row(1, "source", mutable.WrappedArray.newBuilder[Int].+=(1, 2, 3)))
sql("DROP TABLE IF EXISTS create_source")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
index 470e0bf..026c5ca 100644
--- a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
+++ b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
@@ -994,7 +994,7 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
var i = 0
while (i < 11) {
- val array = Array[String](s"name$i", s"$i" + "$" +s"$i.${i}12")
+ val array = Array[String](s"name$i", s"$i" + "\001" +s"$i.${i}12")
writer.write(array)
i += 1
}
@@ -1124,7 +1124,7 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
var i = 0
while (i < 10) {
- val array = Array[String](s"name$i",s"$i" + "$" + s"${i*2}", s"${i/2}" + "$" + s"${i/3}")
+ val array = Array[String](s"name$i",s"$i" + "\001" + s"${i*2}", s"${i/2}" + "\001" + s"${i/3}")
writer.write(array)
i += 1
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c597215/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
index 7c1265c..c7c0d2c 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
@@ -251,7 +251,7 @@ class CarbonDataSourceSuite extends Spark2QueryTest with BeforeAndAfterAll {
sql("drop table if exists create_source")
sql("create table create_source(intField int, stringField string, complexField array<int>) " +
"USING org.apache.spark.sql.CarbonSource OPTIONS('bucketnumber'='1', 'bucketcolumns'='stringField', 'tableName'='create_source')")
- sql("""insert into create_source values(1,"source","1$2$3")""")
+ sql("""insert into create_source values(1,"source","1\0012\0013")""")
checkAnswer(sql("select * from create_source"), Row(1,"source", mutable.WrappedArray.newBuilder[Int].+=(1,2,3)))
sql("drop table if exists create_source")
}
[04/21] carbondata git commit: [CARBONDATA-3141] Removed Carbon Table
Detail Command Test case
Posted by ra...@apache.org.
[CARBONDATA-3141] Removed Carbon Table Detail Command Test case
Removed Carbon Table Detail Command Test case, as this is not used.
This closes #2968
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/216d1c5b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/216d1c5b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/216d1c5b
Branch: refs/heads/branch-1.5
Commit: 216d1c5b267167b07aedbfb2ed94a7ad54584592
Parents: 523515c
Author: praveenmeenakshi56 <pr...@gmail.com>
Authored: Fri Nov 30 18:05:08 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:56:14 2018 +0530
----------------------------------------------------------------------
.../schema/CarbonGetTableDetailCommand.scala | 61 --------------------
.../CarbonGetTableDetailComandTestCase.scala | 56 ------------------
2 files changed, 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/216d1c5b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonGetTableDetailCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonGetTableDetailCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonGetTableDetailCommand.scala
deleted file mode 100644
index 90da68a..0000000
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonGetTableDetailCommand.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.command.schema
-
-import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
-import org.apache.spark.sql.execution.command.DataCommand
-import org.apache.spark.sql.types.{LongType, StringType}
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * collect dynamic detail information of the table, including table size, last modified time, etc.
- */
-case class CarbonGetTableDetailCommand(
- databaseName: String,
- tableNames: Option[Seq[String]])
- extends DataCommand {
-
- override def processData(sparkSession: SparkSession): Seq[Row] = {
- if (tableNames.isDefined) {
- tableNames.get.map { tablename =>
- val carbonTable = CarbonEnv.getCarbonTable(Option(databaseName),
- tablename)(sparkSession)
-
- Row(
- tablename,
- carbonTable.size,
- SegmentStatusManager
- .getTableStatusLastModifiedTime(carbonTable.getAbsoluteTableIdentifier))
- }
- } else {
- Seq.empty[Row]
- }
- }
-
- override def output: Seq[Attribute] = {
- Seq(AttributeReference("table name", StringType, nullable = false)(),
- AttributeReference("table size", LongType, nullable = false)(),
- AttributeReference("last modified time", LongType, nullable = false)())
- }
-
- override protected def opName: String = "GET TABLE DETAIL"
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/216d1c5b/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala
deleted file mode 100644
index 908bc75..0000000
--- a/integration/spark2/src/test/scala/org/apache/spark/sql/CarbonGetTableDetailComandTestCase.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.execution.QueryExecution
-import org.apache.spark.sql.execution.command.schema.CarbonGetTableDetailCommand
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-class CarbonGetTableDetailCommandTestCase extends QueryTest with BeforeAndAfterAll {
- override def beforeAll(): Unit = {
- sql("drop table if exists table_info1")
- sql("create table table_info1 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) stored by 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')")
- sql(s"""load data local inpath '$resourcesPath/data.csv' into table table_info1 options('delimiter'=',', 'quotechar'='\"', 'fileheader'='')""")
-
- sql("drop table if exists table_info2")
- sql("create table table_info2 (empno int, workgroupcategory string, deptno int, projectcode int, attendance int) stored by 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false')")
- sql(s"""load data local inpath '$resourcesPath/data.csv' into table table_info2 options('delimiter'=',', 'quotechar'='\"', 'fileheader'='')""")
- }
-
- ignore("collect the information of tables") {
- val logicalPlan = CarbonGetTableDetailCommand("default", Some(Seq("table_info1", "table_info2")))
- val result =new QueryExecution(sqlContext.sparkSession, logicalPlan)
- .executedPlan
- .execute
- .collect
-
- assertResult(2)(result.length)
- assertResult("table_info1")(result(0).getString(0))
- // 2282 is the size of carbon table. Note that since 1.5.0, we add additional compressor name in metadata
- // and more metadata like written_by and version details are added
- assertResult(2267)(result(0).getLong(1))
- assertResult("table_info2")(result(1).getString(0))
- assertResult(2267)(result(1).getLong(1))
- }
-
- override def afterAll: Unit = {
- sql("drop table if exists table_info1")
- sql("drop table if exists table_info2")
- }
-}
[10/21] carbondata git commit: [CARBONDATA-3116] Support set
carbon.query.directQueryOnDataMap.enabled=true
Posted by ra...@apache.org.
[CARBONDATA-3116] Support set carbon.query.directQueryOnDataMap.enabled=true
This PR fix the error：User can query on dataMap after set carbon.query.directQueryOnDataMap.enabled=true.
This closes #2940
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/26a58508
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/26a58508
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/26a58508
Branch: refs/heads/branch-1.5
Commit: 26a585083fb7887d66c85c3ee27fcae205595742
Parents: 7d7d547
Author: xubo245 <xu...@huawei.com>
Authored: Thu Nov 22 00:32:13 2018 +0800
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:33 2018 +0530
----------------------------------------------------------------------
.../preaggregate/TestPreAggCreateCommand.scala | 43 ++++++++++++++++++++
.../sql/optimizer/CarbonLateDecodeRule.scala | 8 ++--
2 files changed, 48 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26a58508/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index 7851bd1..f07c417 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -17,6 +17,7 @@
package org.apache.carbondata.integration.spark.testsuite.preaggregate
+import java.io.File
import java.util
import java.util.concurrent.{Callable, ExecutorService, Executors, TimeUnit}
@@ -486,6 +487,48 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
executorService.shutdown()
}
+ test("support set carbon.query.directQueryOnDataMap.enabled=true") {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val testData = s"$rootPath/integration/spark-common-test/src/test/resources/sample.csv"
+ sql("drop table if exists mainTable")
+ sql(
+ s"""
+ | CREATE TABLE mainTable
+ | (id Int,
+ | name String,
+ | city String,
+ | age Int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$testData'
+ | into table mainTable
+ """.stripMargin)
+
+ sql(
+ s"""
+ | create datamap preagg_sum on table mainTable
+ | using 'preaggregate'
+ | as select id,sum(age) from mainTable group by id
+ """.stripMargin)
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.VALIDATE_DIRECT_QUERY_ON_DATAMAP, "true")
+
+ sql("set carbon.query.directQueryOnDataMap.enabled=true")
+ checkAnswer(sql("select count(*) from maintable_preagg_sum"), Row(4))
+ sql("set carbon.query.directQueryOnDataMap.enabled=false")
+ val exception: Exception = intercept[AnalysisException] {
+ sql("select count(*) from maintable_preagg_sum").collect()
+ }
+ assert(exception.getMessage.contains("Query On DataMap not supported"))
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.VALIDATE_DIRECT_QUERY_ON_DATAMAP, "false")
+ }
+
class QueryTask(query: String) extends Callable[String] {
override def call(): String = {
var result = "SUCCESS"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/26a58508/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
index 48c6377..36eb9ce 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
@@ -119,7 +119,7 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper {
if (validateQuery && isPreAggDataMapExists) {
val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
if (null != carbonSessionInfo) {
- val supportQueryOnDataMap = CarbonEnv.getThreadParam(
+ val supportQueryOnDataMap = CarbonProperties.getInstance.getProperty(
CarbonCommonConstants.SUPPORT_DIRECT_QUERY_ON_DATAMAP,
CarbonCommonConstants.SUPPORT_DIRECT_QUERY_ON_DATAMAP_DEFAULTVALUE).toBoolean
if (!supportQueryOnDataMap) {
@@ -127,8 +127,10 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper {
}
}
}
- if(isThrowException) {
- throw new AnalysisException("Query On DataMap not supported")
+ if (isThrowException) {
+ throw new AnalysisException("Query On DataMap not supported because "
+ + CarbonCommonConstants.SUPPORT_DIRECT_QUERY_ON_DATAMAP + " is false. " +
+ "Please change the value to true by set command or other if you want to query on DataMap.")
}
}
[02/21] carbondata git commit: [CARBONDATA-3143] Fixed local
dictionary in presto
Posted by ra...@apache.org.
[CARBONDATA-3143] Fixed local dictionary in presto
Problem:
Currently, local dictionary columns are not working for presto as it is not handled in the integration layer.
Solution:
Add local dictionary support to presto integration layer.
This closes #2972
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d909482e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d909482e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d909482e
Branch: refs/heads/branch-1.5
Commit: d909482e6d6d178fceaac0c67c064a1f19f870bc
Parents: 3524f51
Author: ravipesala <ra...@gmail.com>
Authored: Mon Dec 3 18:27:33 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:50:23 2018 +0530
----------------------------------------------------------------------
.../presto/CarbonColumnVectorWrapper.java | 2 +-
.../presto/readers/SliceStreamReader.java | 35 +++
.../PrestoAllDataTypeLocalDictTest.scala | 291 +++++++++++++++++++
.../integrationtest/PrestoAllDataTypeTest.scala | 2 +-
.../carbondata/presto/server/PrestoServer.scala | 4 +-
.../presto/util/CarbonDataStoreCreator.scala | 18 +-
6 files changed, 342 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d909482e/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
index a80751f..f001488 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonColumnVectorWrapper.java
@@ -244,7 +244,7 @@ public class CarbonColumnVectorWrapper implements CarbonColumnVector {
}
@Override public CarbonColumnVector getDictionaryVector() {
- return this.columnVector;
+ return this.columnVector.getDictionaryVector();
}
@Override public void putFloats(int rowId, int count, float[] src, int srcIndex) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d909482e/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
index ab270fc..04e5bb3 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/SliceStreamReader.java
@@ -17,14 +17,19 @@
package org.apache.carbondata.presto.readers;
+import java.util.Optional;
+
import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.scan.result.vector.CarbonDictionary;
import org.apache.carbondata.core.scan.result.vector.impl.CarbonColumnVectorImpl;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.block.DictionaryBlock;
+import com.facebook.presto.spi.block.VariableWidthBlock;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.spi.type.VarcharType;
+import io.airlift.slice.Slices;
import static io.airlift.slice.Slices.wrappedBuffer;
@@ -63,6 +68,36 @@ public class SliceStreamReader extends CarbonColumnVectorImpl implements PrestoV
}
}
+ @Override public void setDictionary(CarbonDictionary dictionary) {
+ super.setDictionary(dictionary);
+ if (dictionary == null) {
+ dictionaryBlock = null;
+ return;
+ }
+ boolean[] nulls = new boolean[dictionary.getDictionarySize()];
+ nulls[0] = true;
+ nulls[1] = true;
+ int[] dictOffsets = new int[dictionary.getDictionarySize() + 1];
+ int size = 0;
+ for (int i = 0; i < dictionary.getDictionarySize(); i++) {
+ if (dictionary.getDictionaryValue(i) != null) {
+ dictOffsets[i] = size;
+ size += dictionary.getDictionaryValue(i).length;
+ }
+ }
+ byte[] singleArrayDictValues = new byte[size];
+ for (int i = 0; i < dictionary.getDictionarySize(); i++) {
+ if (dictionary.getDictionaryValue(i) != null) {
+ System.arraycopy(dictionary.getDictionaryValue(i), 0, singleArrayDictValues, dictOffsets[i],
+ dictionary.getDictionaryValue(i).length);
+ }
+ }
+ dictOffsets[dictOffsets.length - 1] = size;
+ dictionaryBlock = new VariableWidthBlock(dictionary.getDictionarySize(),
+ Slices.wrappedBuffer(singleArrayDictValues), dictOffsets, Optional.of(nulls));
+ values = (int[]) ((CarbonColumnVectorImpl) getDictionaryVector()).getDataArray();
+ }
+
@Override public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d909482e/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
new file mode 100644
index 0000000..356c0e8
--- /dev/null
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeLocalDictTest.scala
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.presto.integrationtest
+
+import java.io.File
+import java.sql.Timestamp
+
+import org.apache.hadoop.fs.permission.{FsAction, FsPermission}
+import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.presto.server.PrestoServer
+
+
+class PrestoAllDataTypeLocalDictTest extends FunSuiteLike with BeforeAndAfterAll {
+
+ private val logger = LogServiceFactory
+ .getLogService(classOf[PrestoAllDataTypeLocalDictTest].getCanonicalName)
+
+ private val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ private val storePath = s"$rootPath/integration/presto/target/store"
+ private val systemPath = s"$rootPath/integration/presto/target/system"
+ private val PrestoServer = new PrestoServer
+
+ // Table schema:
+ // +-------------+----------------+-------------+------------+
+ // | Column name | Data type | Column type | Dictionary |
+ // +-------------+----------------+--------------+-----------+
+ // | id | string | dimension | yes |
+ // +-------------+----------------+-------------+------------+
+ // | date | date | dimension | yes |
+ // +-------------+----------------+-------------+------------+
+ // | country | string | dimension | yes |
+ // +-------------+----------------+-------------+-------------
+ // | name | string | dimension | yes |
+ // +-------------+----------------+-------------+-------------
+ // | phonetype | string | dimension | yes |
+ // +-------------+----------------+-------------+-------------
+ // | serialname | string | dimension | true |
+ // +-------------+----------------+-------------+-------------
+ // | bonus |short decimal | measure | false |
+ // +-------------+----------------+-------------+-------------
+ // | monthlyBonus| longdecimal | measure | false |
+ // +-------------+----------------+-------------+-------------
+ // | dob | timestamp | dimension | true |
+ // +-------------+----------------+-------------+------------+
+ // | shortField | shortfield | measure | true |
+ // +-------------+----------------+-------------+-------------
+ // |isCurrentEmp | boolean | measure | true |
+ // +-------------+----------------+-------------+------------+
+
+ override def beforeAll: Unit = {
+ import org.apache.carbondata.presto.util.CarbonDataStoreCreator
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION,
+ systemPath)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME,
+ "Presto")
+ CarbonDataStoreCreator
+ .createCarbonStore(storePath,
+ s"$rootPath/integration/presto/src/test/resources/alldatatype.csv", true)
+ logger.info(s"\nCarbon store is created at location: $storePath")
+ cleanUp
+ PrestoServer.startServer(storePath)
+ }
+
+ override def afterAll(): Unit = {
+ PrestoServer.stopServer()
+ }
+
+ test("select string type with order by clause") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME FROM TESTDB.TESTTABLE ORDER BY NAME")
+ val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "akash"),
+ Map("NAME" -> "anubhav"),
+ Map("NAME" -> "bhavya"),
+ Map("NAME" -> "geetika"),
+ Map("NAME" -> "jatin"),
+ Map("NAME" -> "jitesh"),
+ Map("NAME" -> "liang"),
+ Map("NAME" -> "prince"),
+ Map("NAME" -> "ravindra"),
+ Map("NAME" -> "sahil"),
+ Map("NAME" -> null)
+ )
+ assert(actualResult.equals(expectedResult))
+ }
+
+ test("test and filter clause with greater than expression") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
+ "WHERE BONUS>1234 AND ID>2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
+ "BONUS ORDER BY ID")
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 4,
+ "NAME" -> "prince",
+ "BONUS" -> java.math.BigDecimal.valueOf(9999.9990).setScale(4),
+ "DATE" -> "2015-07-26",
+ "SALARY" -> 15003.0,
+ "SERIALNAME" -> "ASD66902",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone2435"),
+ Map("ID" -> 5,
+ "NAME" -> "bhavya",
+ "BONUS" -> java.math.BigDecimal.valueOf(5000.999).setScale(4),
+ "DATE" -> "2015-07-27",
+ "SALARY" -> 15004.0,
+ "SERIALNAME" -> "ASD90633",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone2441"))
+ assert(actualResult.toString() equals expectedResult.toString())
+
+
+ }
+
+ test("test and filter clause with greater than equal to expression") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
+ "WHERE BONUS>=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
+ "BONUS ORDER BY ID")
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1,
+ "NAME" -> "anubhav",
+ "BONUS" -> java.math.BigDecimal.valueOf(1234.4440).setScale(4),
+ "DATE" -> "2015-07-23",
+ "SALARY" -> "5000000.0",
+ "SERIALNAME" -> "ASD69643",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone197"),
+ Map("ID" -> 2,
+ "NAME" -> "jatin",
+ "BONUS" -> java.math.BigDecimal.valueOf(1234.5555).setScale(4)
+ ,
+ "DATE" -> "2015-07-24",
+ "SALARY" -> java.math.BigDecimal.valueOf(150010.9990).setScale(3),
+ "SERIALNAME" -> "ASD42892",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone756"),
+ Map("ID" -> 4,
+ "NAME" -> "prince",
+ "BONUS" -> java.math.BigDecimal.valueOf(9999.9990).setScale(4),
+ "DATE" -> "2015-07-26",
+ "SALARY" -> java.math.BigDecimal.valueOf(15003.0).setScale(1),
+ "SERIALNAME" -> "ASD66902",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone2435"),
+ Map("ID" -> 5,
+ "NAME" -> "bhavya",
+ "BONUS" -> java.math.BigDecimal.valueOf(5000.9990).setScale(4),
+ "DATE" -> "2015-07-27",
+ "SALARY" -> java.math.BigDecimal.valueOf(15004.0).setScale(1),
+ "SERIALNAME" -> "ASD90633",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone2441"))
+ assert(actualResult.toString() equals expectedResult.toString())
+ }
+ test("test and filter clause with less than equal to expression") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
+ "WHERE BONUS<=1234.444 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
+ "BONUS ORDER BY ID LIMIT 2")
+
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1,
+ "NAME" -> "anubhav",
+ "BONUS" -> java.math.BigDecimal.valueOf(1234.4440).setScale(4),
+ "DATE" -> "2015-07-23",
+ "SALARY" -> "5000000.0",
+ "SERIALNAME" -> "ASD69643",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone197"),
+ Map("ID" -> 3,
+ "NAME" -> "liang",
+ "BONUS" -> java.math.BigDecimal.valueOf(600.7770).setScale(4),
+ "DATE" -> "2015-07-25",
+ "SALARY" -> java.math.BigDecimal.valueOf(15002.11).setScale(2),
+ "SERIALNAME" -> "ASD37014",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone1904"))
+ assert(actualResult.toString() equals expectedResult.toString())
+ }
+ test("test equal to expression on decimal value") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID FROM TESTDB.TESTTABLE WHERE BONUS=1234.444")
+
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1))
+
+ assert(actualResult equals expectedResult)
+ }
+ test("test less than expression with and operator") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY,BONUS FROM TESTDB.TESTTABLE " +
+ "WHERE BONUS>1234 AND ID<2 GROUP BY ID,DATE,COUNTRY,NAME,PHONETYPE,SERIALNAME,SALARY," +
+ "BONUS ORDER BY ID")
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 1,
+ "NAME" -> "anubhav",
+ "BONUS" -> java.math.BigDecimal.valueOf(1234.4440).setScale(4),
+ "DATE" -> "2015-07-23",
+ "SALARY" -> 5000000.0,
+ "SERIALNAME" -> "ASD69643",
+ "COUNTRY" -> "china",
+ "PHONETYPE" -> "phone197"))
+ assert(actualResult.toString().equals(expectedResult.toString()))
+ }
+ test("test the result for in clause") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME from testdb.testtable WHERE PHONETYPE IN('phone1848','phone706')")
+ val expectedResult: List[Map[String, Any]] = List(
+ Map("NAME" -> "geetika"),
+ Map("NAME" -> "ravindra"),
+ Map("NAME" -> "jitesh"))
+
+ assert(actualResult.equals(expectedResult))
+ }
+ test("test the result for not in clause") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery(
+ "SELECT NAME from testdb.testtable WHERE PHONETYPE NOT IN('phone1848','phone706')")
+ val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "anubhav"),
+ Map("NAME" -> "jatin"),
+ Map("NAME" -> "liang"),
+ Map("NAME" -> "prince"),
+ Map("NAME" -> "bhavya"),
+ Map("NAME" -> "akash"),
+ Map("NAME" -> "sahil"))
+
+ assert(actualResult.equals(expectedResult))
+ }
+ test("test for null operator on date data type") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT ID FROM TESTDB.TESTTABLE WHERE DATE IS NULL")
+ val expectedResult: List[Map[String, Any]] = List(Map("ID" -> 9),Map("ID" -> null))
+ assert(actualResult.equals(expectedResult))
+
+ }
+ test("test for not null operator on date data type") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DATE IS NOT NULL AND ID=9")
+ val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"))
+ assert(actualResult.equals(expectedResult))
+
+ }
+ test("test for not null operator on timestamp type") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME FROM TESTDB.TESTTABLE WHERE DOB IS NOT NULL AND ID=9")
+ val expectedResult: List[Map[String, Any]] = List(Map("NAME" -> "ravindra"),
+ Map("NAME" -> "jitesh"))
+ assert(actualResult.equals(expectedResult))
+
+ }
+
+ test("test timestamp datatype using cast operator") {
+ val actualResult: List[Map[String, Any]] = PrestoServer
+ .executeQuery("SELECT NAME AS RESULT FROM TESTDB.TESTTABLE WHERE DOB = CAST('2016-04-14 15:00:09' AS TIMESTAMP)")
+ val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> "jatin"))
+ assert(actualResult.equals(expectedResult))
+ }
+
+ private def cleanUp(): Unit = {
+ FileFactory.deleteFile(s"$storePath/Fact", FileType.LOCAL)
+ FileFactory
+ .createDirectoryAndSetPermission(s"$storePath/_system",
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ FileFactory
+ .createDirectoryAndSetPermission(s"$storePath/.DS_Store",
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ FileFactory.createNewFile(s"$storePath/testdb/.DS_STORE",FileType.LOCAL)
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d909482e/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
index d6e322b..97539c0 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
@@ -40,7 +40,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
+ "../../../..").getCanonicalPath
private val storePath = s"$rootPath/integration/presto/target/store"
private val systemPath = s"$rootPath/integration/presto/target/system"
-
+ private val PrestoServer = new PrestoServer
// Table schema:
// +-------------+----------------+-------------+------------+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d909482e/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
index 2f3b8f4..1b1accf 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala
@@ -35,7 +35,7 @@ import org.slf4j.{Logger, LoggerFactory}
import org.apache.carbondata.core.util.ThreadLocalSessionInfo
import org.apache.carbondata.presto.CarbondataPlugin
-object PrestoServer {
+class PrestoServer {
val CARBONDATA_CATALOG = "carbondata"
val CARBONDATA_CONNECTOR = "carbondata"
@@ -45,7 +45,7 @@ object PrestoServer {
val prestoProperties: util.Map[String, String] = Map(("http-server.http.port", "8086")).asJava
createSession
- val queryRunner = new DistributedQueryRunner(createSession, 4, prestoProperties)
+ lazy val queryRunner = new DistributedQueryRunner(createSession, 4, prestoProperties)
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d909482e/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index 4b973a1..f687855 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -67,7 +67,8 @@ object CarbonDataStoreCreator {
/**
* Create store without any restructure
*/
- def createCarbonStore(storePath: String, dataFilePath: String): Unit = {
+ def createCarbonStore(storePath: String, dataFilePath: String,
+ useLocalDict: Boolean = false): Unit = {
try {
logger.info("Creating The Carbon Store")
val dbName: String = "testdb"
@@ -80,7 +81,7 @@ object CarbonDataStoreCreator {
// val factFilePath: String = new File(dataFilePath).getCanonicalPath
val storeDir: File = new File(absoluteTableIdentifier.getTablePath)
CarbonUtil.deleteFoldersAndFiles(storeDir)
- val table: CarbonTable = createTable(absoluteTableIdentifier)
+ val table: CarbonTable = createTable(absoluteTableIdentifier, useLocalDict)
writeDictionary(dataFilePath, table, absoluteTableIdentifier)
val schema: CarbonDataLoadSchema = new CarbonDataLoadSchema(table)
val loadModel: CarbonLoadModel = new CarbonLoadModel()
@@ -141,7 +142,8 @@ object CarbonDataStoreCreator {
}
}
- private def createTable(absoluteTableIdentifier: AbsoluteTableIdentifier): CarbonTable = {
+ private def createTable(absoluteTableIdentifier: AbsoluteTableIdentifier,
+ useLocalDict: Boolean): CarbonTable = {
val tableInfo: TableInfo = new TableInfo()
tableInfo.setTablePath(absoluteTableIdentifier.getTablePath)
tableInfo.setDatabaseName(
@@ -151,7 +153,9 @@ object CarbonDataStoreCreator {
absoluteTableIdentifier.getCarbonTableIdentifier.getTableName)
val columnSchemas = new ArrayList[ColumnSchema]()
val dictionaryEncoding: ArrayList[Encoding] = new ArrayList[Encoding]()
- dictionaryEncoding.add(Encoding.DICTIONARY)
+ if (!useLocalDict) {
+ dictionaryEncoding.add(Encoding.DICTIONARY)
+ }
val invertedIndexEncoding: ArrayList[Encoding] = new ArrayList[Encoding]()
invertedIndexEncoding.add(Encoding.INVERTED_INDEX)
@@ -225,9 +229,9 @@ object CarbonDataStoreCreator {
val salary: ColumnSchema = new ColumnSchema()
salary.setColumnName("salary")
salary.setDataType(DataTypes.DOUBLE)
- salary.setEncodingList(dictionaryEncoding)
+ salary.setEncodingList(new util.ArrayList[Encoding]())
salary.setColumnUniqueId(UUID.randomUUID().toString)
- salary.setDimensionColumn(true)
+ salary.setDimensionColumn(false)
salary.setSchemaOrdinal(6)
salary.setColumnReferenceId(salary.getColumnUniqueId)
columnSchemas.add(salary)
@@ -291,6 +295,8 @@ object CarbonDataStoreCreator {
new util.ArrayList[SchemaEvolutionEntry]())
tableSchema.setSchemaEvolution(schemaEvol)
tableSchema.setTableId(UUID.randomUUID().toString)
+ tableSchema.getTableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE,
+ String.valueOf(useLocalDict))
tableInfo.setTableUniqueName(
absoluteTableIdentifier.getCarbonTableIdentifier.getTableUniqueName
)
[05/21] carbondata git commit: [CARBONDATA-3147] Fixed concurrent
load issue
Posted by ra...@apache.org.
[CARBONDATA-3147] Fixed concurrent load issue
Problem: During datamap commit, tablestatus_UUID files were being renamed to tablestatus due to which any new in progress entry was being skipped and eventually during writing of success status 'Entry not found exception was thrown'.
Solution: Instead of renaming the files, now we are reading both the files and merging the contents. Then the updated details are written to the tablestatus file.
This closes #2977
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7d7d547e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7d7d547e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7d7d547e
Branch: refs/heads/branch-1.5
Commit: 7d7d547eff033e8760dff59b7c0a5fece76657c7
Parents: 8f57294
Author: kunal642 <ku...@gmail.com>
Authored: Wed Dec 5 16:00:41 2018 +0530
Committer: Raghunandan S <ca...@gmail.com>
Committed: Mon Dec 17 18:58:33 2018 +0530
----------------------------------------------------------------------
.../spark/rdd/AggregateDataMapCompactor.scala | 69 ++++++---
.../spark/rdd/CarbonDataRDDFactory.scala | 3 +-
.../management/CarbonLoadDataCommand.scala | 2 +-
.../preaaggregate/PreAggregateListeners.scala | 142 ++++++++++++++-----
.../preaaggregate/PreAggregateUtil.scala | 7 +-
.../processing/util/CarbonLoaderUtil.java | 10 +-
6 files changed, 174 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d7d547e/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
index 2119e4c..6d28295 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
@@ -23,11 +23,12 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.{CarbonSession, SQLContext}
import org.apache.spark.sql.execution.command.CompactionModel
import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
-import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
+import org.apache.spark.sql.execution.command.preaaggregate.{CommitPreAggregateListener, PreAggregateUtil}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.Segment
-import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.OperationContext
@@ -79,20 +80,32 @@ class AggregateDataMapCompactor(carbonLoadModel: CarbonLoadModel,
CarbonSession.threadSet(CarbonCommonConstants.SUPPORT_DIRECT_QUERY_ON_DATAMAP,
"true")
loadCommand.processData(sqlContext.sparkSession)
- val newLoadMetaDataDetails = SegmentStatusManager.readLoadMetadata(
+ // After load is completed for child table the UUID table status will have 0.1 as success
+ // and the table status file will have 0,1,2,3 as Success and 0.1 as In Progress.
+ // Therefore we will read the table status and write 0,1,2,3 as Compacted as the commit
+ // listener will take care of merging the UUID and the table status files.
+ val newMetadataDetails = SegmentStatusManager.readLoadMetadata(
carbonTable.getMetadataPath, uuid)
- val updatedLoadMetaDataDetails = newLoadMetaDataDetails collect {
- case load if loadMetaDataDetails.contains(load) =>
- load.setMergedLoadName(mergedLoadName)
- load.setSegmentStatus(SegmentStatus.COMPACTED)
- load.setModificationOrdeletionTimesStamp(System.currentTimeMillis())
- load
- case other => other
- }
+ val mergedContent = loadMetaDataDetails.asScala.map {
+ segment => segment.setSegmentStatus(SegmentStatus.COMPACTED)
+ segment.setMergedLoadName(mergedLoadName)
+ segment.setModificationOrdeletionTimesStamp(CarbonUpdateUtil.readCurrentTime)
+ segment
+ } ++ newMetadataDetails
SegmentStatusManager.writeLoadDetailsIntoFile(
CarbonTablePath.getTableStatusFilePathWithUUID(carbonTable.getTablePath, uuid),
- updatedLoadMetaDataDetails)
- carbonLoadModel.setLoadMetadataDetails(updatedLoadMetaDataDetails.toList.asJava)
+ mergedContent.toArray)
+ carbonLoadModel.setLoadMetadataDetails((carbonLoadModel.getLoadMetadataDetails.asScala ++
+ newMetadataDetails).asJava)
+ // If isCompaction is true then it means that the compaction on aggregate table was
+ // triggered by the maintable thus no need to commit the tablestatus file but if the
+ // compaction was triggered directly for aggregate table then commit has to be fired as
+ // the commit listener would not be called.
+ val directAggregateCompactionCall = Option(operationContext
+ .getProperty("isCompaction")).getOrElse("false").toString.toBoolean
+ if (!directAggregateCompactionCall) {
+ commitAggregateTableStatus(carbonTable, uuid)
+ }
} finally {
// check if any other segments needs compaction on in case of MINOR_COMPACTION.
// For example: after 8.1 creation 0.1, 4.1, 8.1 have to be merged to 0.2 if threshhold
@@ -112,12 +125,7 @@ class AggregateDataMapCompactor(carbonLoadModel: CarbonLoadModel,
if (!compactionModel.compactionType.equals(CompactionType.MAJOR) &&
!compactionModel.compactionType.equals(CompactionType.CUSTOM)) {
if (!identifySegmentsToBeMerged().isEmpty) {
- val uuidTableStaus = CarbonTablePath.getTableStatusFilePathWithUUID(
- carbonTable.getTablePath, uuid)
- val tableStatus = CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath)
- if (!uuidTableStaus.equalsIgnoreCase(tableStatus)) {
- FileFactory.getCarbonFile(uuidTableStaus).renameForce(tableStatus)
- }
+ commitAggregateTableStatus(carbonTable, uuid)
executeCompaction()
}
}
@@ -133,4 +141,27 @@ class AggregateDataMapCompactor(carbonLoadModel: CarbonLoadModel,
}
}
}
+
+ /**
+ * Used to merge the contents of tablestatus and tablestatus_uuid files and write the new
+ * details to tablestatus file. For Example:-
+ * tablestatus contents are = 0(Success), 1(Success),2(Success),3(Success), 0.1(In Progress)
+ * tablestatus_uuid contents are = 0(Compacted), 1(Compacted),2(Compacted),3(Compacted), 0.1
+ * (Success).
+ *
+ * So after merging the tablestatus file will have: 0(Compacted), 1(Compacted),2(Compacted),
+ * 3(Compacted), 0.1(Success).
+ *
+ * NOTE: This method will be called when direct compaction is fired on child aggregate table or
+ * when there are anymore segments to be compacted and the intermediate state of the
+ * tablestatus has to be committed for further compaction to pick other segments.
+ */
+ private def commitAggregateTableStatus(carbonTable: CarbonTable, uuid: String) {
+ if (!CommitPreAggregateListener.mergeTableStatusContents(carbonTable, CarbonTablePath
+ .getTableStatusFilePathWithUUID(carbonTable.getTablePath, uuid), CarbonTablePath
+ .getTableStatusFilePathWithUUID(carbonTable.getTablePath, ""))) {
+ throw new RuntimeException("Unable to acquire lock for table status updation")
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d7d547e/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 7e2e7d9..b07be72 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -461,7 +461,8 @@ object CarbonDataRDDFactory {
}
return null
}
- val uniqueTableStatusId = operationContext.getProperty("uuid").asInstanceOf[String]
+ val uniqueTableStatusId = Option(operationContext.getProperty("uuid")).getOrElse("")
+ .asInstanceOf[String]
if (loadStatus == SegmentStatus.LOAD_FAILURE) {
// update the load entry in table status file for changing the status to marked for delete
CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel, uniqueTableStatusId)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d7d547e/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index af406bb..6a8a9cf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -349,7 +349,7 @@ case class CarbonLoadDataCommand(
case ex: Exception =>
LOGGER.error(ex)
// update the load entry in table status file for changing the status to marked for delete
- if (isUpdateTableStatusRequired) {
+ if (isUpdateTableStatusRequired && !table.isChildDataMap) {
CarbonLoaderUtil.updateTableStatusForFailure(carbonLoadModel, uuid)
}
throw ex
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d7d547e/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
index f606c04..3038c93 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
@@ -31,9 +31,11 @@ import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.common.exceptions.MetadataProcessException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.{DataMapStoreManager, Segment}
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock}
import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable}
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.CarbonUtil
@@ -69,16 +71,32 @@ trait CommitHelper {
protected def markInProgressSegmentAsDeleted(tableStatusFile: String,
operationContext: OperationContext,
carbonTable: CarbonTable): Unit = {
- val loadMetaDataDetails = SegmentStatusManager.readTableStatusFile(tableStatusFile)
- val segmentBeingLoaded =
- operationContext.getProperty(carbonTable.getTableUniqueName + "_Segment").toString
- val newDetails = loadMetaDataDetails.collect {
- case detail if detail.getLoadName.equalsIgnoreCase(segmentBeingLoaded) =>
- detail.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE)
- detail
- case others => others
+ val lockFile: ICarbonLock = new SegmentStatusManager(carbonTable
+ .getAbsoluteTableIdentifier).getTableStatusLock
+ val retryCount = CarbonLockUtil
+ .getLockProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK,
+ CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK_DEFAULT)
+ val maxTimeout = CarbonLockUtil
+ .getLockProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK,
+ CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK_DEFAULT)
+ try {
+ if (lockFile.lockWithRetries(retryCount, maxTimeout)) {
+ val loadMetaDataDetails = SegmentStatusManager.readTableStatusFile(tableStatusFile)
+ val segmentBeingLoaded =
+ operationContext.getProperty(carbonTable.getTableUniqueName + "_Segment").toString
+ val newDetails = loadMetaDataDetails.collect {
+ case detail if detail.getLoadName.equalsIgnoreCase(segmentBeingLoaded) =>
+ detail.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE)
+ detail
+ case others => others
+ }
+ SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusFile, newDetails)
+ } else {
+ throw new RuntimeException("Uable to update table status file")
+ }
+ } finally {
+ lockFile.unlock()
}
- SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusFile, newDetails)
}
/**
@@ -111,6 +129,43 @@ trait CommitHelper {
}
}
+ def mergeTableStatusContents(carbonTable: CarbonTable, uuidTableStatusPath: String,
+ tableStatusPath: String): Boolean = {
+ val lockFile: ICarbonLock = new SegmentStatusManager(carbonTable
+ .getAbsoluteTableIdentifier).getTableStatusLock
+ try {
+ val retryCount = CarbonLockUtil
+ .getLockProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK,
+ CarbonCommonConstants.NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK_DEFAULT)
+ val maxTimeout = CarbonLockUtil
+ .getLockProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK,
+ CarbonCommonConstants.MAX_TIMEOUT_FOR_CONCURRENT_LOCK_DEFAULT)
+ if (lockFile.lockWithRetries(retryCount, maxTimeout)) {
+ val tableStatusContents = SegmentStatusManager.readTableStatusFile(tableStatusPath)
+ val newLoadContent = SegmentStatusManager.readTableStatusFile(uuidTableStatusPath)
+ val mergedContent = tableStatusContents.collect {
+ case content =>
+ val contentIndex = newLoadContent.indexOf(content)
+ if (contentIndex == -1) {
+ content
+ } else {
+ newLoadContent(contentIndex)
+ }
+ }
+ SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, mergedContent)
+ true
+ } else {
+ false
+ }
+ } catch {
+ case ex: Exception =>
+ LOGGER.error("Exception occurred while merging files", ex)
+ false
+ } finally {
+ lockFile.unlock()
+ }
+ }
+
/**
* Used to remove table status files with UUID and segment folders.
*/
@@ -156,20 +211,19 @@ object AlterTableDropPartitionPostStatusListener extends OperationEventListener
// Generate table status file name without UUID, forExample: tablestatus
val newTableSchemaPath = CarbonTablePath.getTableStatusFilePath(
childCarbonTable.getTablePath)
- renameDataMapTableStatusFiles(oldTableSchemaPath, newTableSchemaPath, uuid)
+ mergeTableStatusContents(childCarbonTable, oldTableSchemaPath, newTableSchemaPath)
}
// if true then the commit for one of the child tables has failed
val commitFailed = renamedDataMaps.lengthCompare(childCommands.length) != 0
if (commitFailed) {
- LOGGER.info("Reverting table status file to original state")
- renamedDataMaps.foreach {
- command =>
- val carbonTable = command.table
- // rename the backup tablestatus i.e tablestatus_backup_UUID to tablestatus
- val backupTableSchemaPath =
- CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath) + "_backup_" + uuid
- val tableSchemaPath = CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath)
- renameDataMapTableStatusFiles(backupTableSchemaPath, tableSchemaPath, "")
+ LOGGER.warn("Reverting table status file to original state")
+ childCommands.foreach {
+ childDropCommand =>
+ val tableStatusPath = CarbonTablePath.getTableStatusFilePath(
+ childDropCommand.table.getTablePath)
+ markInProgressSegmentAsDeleted(tableStatusPath,
+ operationContext,
+ childDropCommand.table)
}
}
commitFailed
@@ -200,7 +254,6 @@ object AlterTableDropPartitionMetaListener extends OperationEventListener{
val dropPartitionEvent = event.asInstanceOf[AlterTableDropPartitionMetaEvent]
val parentCarbonTable = dropPartitionEvent.parentCarbonTable
val partitionsToBeDropped = dropPartitionEvent.specs.flatMap(_.keys)
- val sparkSession = SparkSession.getActiveSession.get
if (parentCarbonTable.hasAggregationDataMap) {
// used as a flag to block direct drop partition on aggregate tables fired by the user
operationContext.setProperty("isInternalDropCall", "true")
@@ -325,7 +378,7 @@ object CompactionProcessMetaListener extends OperationEventListener {
childDataFrame,
false,
sparkSession)
- val uuid = Option(operationContext.getProperty("uuid")).getOrElse("").toString
+ val uuid = Option(operationContext.getProperty("uuid")).getOrElse(UUID.randomUUID()).toString
loadCommand.processMetadata(sparkSession)
operationContext.setProperty(table.getTableName + "_Compaction", loadCommand)
operationContext.setProperty("uuid", uuid)
@@ -460,7 +513,7 @@ object LoadPostAggregateListener extends OperationEventListener {
.asInstanceOf[mutable.ArrayBuffer[AggregationDataMapSchema]]
// sorting the datamap for timeseries rollup
val sortedList = aggregationDataMapList.sortBy(_.getOrdinal)
- for (dataMapSchema: AggregationDataMapSchema <- sortedList) {
+ val successDataMaps = sortedList.takeWhile { dataMapSchema =>
val childLoadCommand = operationContext
.getProperty(dataMapSchema.getChildSchema.getTableName)
.asInstanceOf[CarbonLoadDataCommand]
@@ -493,9 +546,32 @@ object LoadPostAggregateListener extends OperationEventListener {
isOverwrite,
sparkSession)
}
+ val loadFailed = successDataMaps.lengthCompare(sortedList.length) != 0
+ if (loadFailed) {
+ successDataMaps.foreach(dataMapSchema => markSuccessSegmentsAsFailed(operationContext
+ .getProperty(dataMapSchema.getChildSchema.getTableName)
+ .asInstanceOf[CarbonLoadDataCommand]))
+ throw new RuntimeException(
+ "Data Load failed for DataMap. Please check logs for the failure")
+ }
}
}
}
+
+ private def markSuccessSegmentsAsFailed(childLoadCommand: CarbonLoadDataCommand) {
+ val segmentToRevert = childLoadCommand.operationContext
+ .getProperty(childLoadCommand.table.getTableUniqueName + "_Segment")
+ val tableStatusPath = CarbonTablePath.getTableStatusFilePath(
+ childLoadCommand.table.getTablePath)
+ val tableStatusContents = SegmentStatusManager.readTableStatusFile(tableStatusPath)
+ val updatedLoadDetails = tableStatusContents.collect {
+ case content if content.getLoadName == segmentToRevert =>
+ content.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE)
+ content
+ case others => others
+ }
+ SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, updatedLoadDetails)
+ }
}
/**
@@ -540,7 +616,7 @@ object CommitPreAggregateListener extends OperationEventListener with CommitHelp
operationContext.getProperty(dataMapSchema.getChildSchema.getTableName + "_Compaction")
.asInstanceOf[CarbonLoadDataCommand]
}
- }
+ }
var commitFailed = false
try {
if (dataMapSchemas.nonEmpty) {
@@ -554,23 +630,19 @@ object CommitPreAggregateListener extends OperationEventListener with CommitHelp
// Generate table status file name without UUID, forExample: tablestatus
val newTableSchemaPath = CarbonTablePath.getTableStatusFilePath(
childCarbonTable.getTablePath)
- renameDataMapTableStatusFiles(oldTableSchemaPath, newTableSchemaPath, uuid)
+ mergeTableStatusContents(childCarbonTable, oldTableSchemaPath, newTableSchemaPath)
}
// if true then the commit for one of the child tables has failed
commitFailed = renamedDataMaps.lengthCompare(dataMapSchemas.length) != 0
if (commitFailed) {
LOGGER.warn("Reverting table status file to original state")
- renamedDataMaps.foreach {
- loadCommand =>
- val carbonTable = loadCommand.table
- // rename the backup tablestatus i.e tablestatus_backup_UUID to tablestatus
- val backupTableSchemaPath =
- CarbonTablePath.getTableStatusFilePath(carbonTable.getTablePath) + "_backup_" +
- uuid
- val tableSchemaPath = CarbonTablePath
- .getTableStatusFilePath(carbonTable.getTablePath)
- markInProgressSegmentAsDeleted(backupTableSchemaPath, operationContext, carbonTable)
- renameDataMapTableStatusFiles(backupTableSchemaPath, tableSchemaPath, "")
+ childLoadCommands.foreach {
+ childLoadCommand =>
+ val tableStatusPath = CarbonTablePath.getTableStatusFilePath(
+ childLoadCommand.table.getTablePath)
+ markInProgressSegmentAsDeleted(tableStatusPath,
+ operationContext,
+ childLoadCommand.table)
}
}
// after success/failure of commit delete all tablestatus files with UUID in their names.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d7d547e/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index d16f570..0599fb3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -587,7 +587,7 @@ object PreAggregateUtil {
validateSegments: Boolean,
loadCommand: CarbonLoadDataCommand,
isOverwrite: Boolean,
- sparkSession: SparkSession): Unit = {
+ sparkSession: SparkSession): Boolean = {
CarbonSession.threadSet(
CarbonCommonConstants.CARBON_INPUT_SEGMENTS +
parentTableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase) + "." +
@@ -601,6 +601,11 @@ object PreAggregateUtil {
"true")
try {
loadCommand.processData(sparkSession)
+ true
+ } catch {
+ case ex: Exception =>
+ LOGGER.error("Data Load failed for DataMap: ", ex)
+ false
} finally {
CarbonSession.threadUnset(
CarbonCommonConstants.CARBON_INPUT_SEGMENTS +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d7d547e/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 2563768..64fcaa2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -339,8 +339,14 @@ public final class CarbonLoaderUtil {
}
}
- SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetails
- .toArray(new LoadMetadataDetails[listOfLoadFolderDetails.size()]));
+ if (loadModel.getCarbonDataLoadSchema().getCarbonTable().isChildDataMap() && !loadStartEntry
+ && !uuid.isEmpty() && segmentsToBeDeleted.isEmpty() && !insertOverwrite) {
+ SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath,
+ new LoadMetadataDetails[] { newMetaEntry });
+ } else {
+ SegmentStatusManager.writeLoadDetailsIntoFile(tableStatusPath, listOfLoadFolderDetails
+ .toArray(new LoadMetadataDetails[listOfLoadFolderDetails.size()]));
+ }
// Delete all old stale segment folders
for (CarbonFile staleFolder : staleFolders) {
// try block is inside for loop because even if there is failure in deletion of 1 stale