You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/01/31 05:22:21 UTC
[01/50] [abbrv] carbondata git commit: [CARBONDATA-2020][Old Store
Support] Add filter support for old store reading to improve query
performance [Forced Update!]
Repository: carbondata
Updated Branches:
refs/heads/carbonstore 3e1da7c3b -> 00cc601da (forced update)
[CARBONDATA-2020][Old Store Support] Add filter support for old store reading to improve query performance
Problem
For old stores blocklet level min/max comparison was not happening in the executor side due to which all the blocklets were getting scanned. This increased the IO and scanning time in the executor.
Solution
Modified code to retrieve the min/max value from blocklet node and use it for comparsion while scanning for valid blocklets.
This closes #1818
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d1d726a7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d1d726a7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d1d726a7
Branch: refs/heads/carbonstore
Commit: d1d726a777c8ed848d71fa05a18dfd70b05d65bd
Parents: 9550e69
Author: manishgupta88 <to...@gmail.com>
Authored: Wed Jan 17 11:27:09 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Jan 18 22:30:23 2018 +0530
----------------------------------------------------------------------
.../blockletindex/BlockletDataMap.java | 8 +++++--
.../BlockletDataRefNodeWrapper.java | 19 ++++++++++++++--
.../core/scan/filter/ColumnFilterInfo.java | 24 ++++++++++++++++++++
.../ImplicitIncludeFilterExecutorImpl.java | 7 +++++-
.../core/scan/scanner/impl/FilterScanner.java | 18 ++++++++++++---
5 files changed, 68 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d1d726a7/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index ee74fad..e1fa686 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -691,8 +691,12 @@ public class BlockletDataMap implements DataMap, Cacheable {
byte[][] minValue, String filePath, int blockletId) {
BitSet bitSet = null;
if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
- String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1)
- + CarbonCommonConstants.FILE_SEPARATOR + blockletId;
+ String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1);
+ // this case will come in case of old store where index file does not contain the
+ // blocklet information
+ if (blockletId != -1) {
+ uniqueBlockPath = uniqueBlockPath + CarbonCommonConstants.FILE_SEPARATOR + blockletId;
+ }
bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
.isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d1d726a7/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
index dfc8a38..1585414 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkRea
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
/**
* wrapper for blocklet data map data
@@ -91,11 +92,25 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
return blockInfos.get(index).getDetailInfo().getBlockletId().toString();
}
- @Override public byte[][] getColumnsMaxValue() {
+ @Override
+ public byte[][] getColumnsMaxValue() {
+ BlockletIndex blockletIndex =
+ blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
+ // In case of blocklet distribution this will be null
+ if (null != blockletIndex) {
+ return blockletIndex.getMinMaxIndex().getMaxValues();
+ }
return null;
}
- @Override public byte[][] getColumnsMinValue() {
+ @Override
+ public byte[][] getColumnsMinValue() {
+ BlockletIndex blockletIndex =
+ blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
+ // In case of blocklet distribution this will be null
+ if (null != blockletIndex) {
+ return blockletIndex.getMinMaxIndex().getMinValues();
+ }
return null;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d1d726a7/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
index 6a7fced..b5b6017 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
@@ -18,7 +18,11 @@
package org.apache.carbondata.core.scan.filter;
import java.io.Serializable;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
public class ColumnFilterInfo implements Serializable {
@@ -32,6 +36,7 @@ public class ColumnFilterInfo implements Serializable {
* Implicit column filter values to be used for block and blocklet pruning
*/
private List<String> implicitColumnFilterList;
+ private transient Set<String> implicitDriverColumnFilterList;
private List<Integer> excludeFilterList;
/**
* maintain the no dictionary filter values list.
@@ -95,4 +100,23 @@ public class ColumnFilterInfo implements Serializable {
public void setMeasuresFilterValuesList(List<Object> measuresFilterValuesList) {
this.measuresFilterValuesList = measuresFilterValuesList;
}
+
+ public Set<String> getImplicitDriverColumnFilterList() {
+ // this list is required to be populated only n case of driver, so in executor this check will
+ // avoid unnecessary loading of the driver filter list
+ if (null == implicitDriverColumnFilterList) {
+ populateBlockIdListForDriverBlockPruning();
+ }
+ return implicitDriverColumnFilterList;
+ }
+
+ private void populateBlockIdListForDriverBlockPruning() {
+ implicitDriverColumnFilterList = new HashSet<>(implicitColumnFilterList.size());
+ String blockId = null;
+ for (String blockletId : implicitColumnFilterList) {
+ blockId =
+ blockletId.substring(0, blockletId.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR));
+ implicitDriverColumnFilterList.add(blockId);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d1d726a7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
index 5289e45..9c4c7ba 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
@@ -70,7 +70,12 @@ public class ImplicitIncludeFilterExecutorImpl
BitSet bitSet = new BitSet(1);
boolean isScanRequired = false;
String shortBlockId = CarbonTablePath.getShortBlockId(uniqueBlockPath);
- if (dimColumnEvaluatorInfo.getFilterValues().getImplicitColumnFilterList()
+ if (uniqueBlockPath.endsWith(".carbondata")) {
+ if (dimColumnEvaluatorInfo.getFilterValues().getImplicitDriverColumnFilterList()
+ .contains(shortBlockId)) {
+ isScanRequired = true;
+ }
+ } else if (dimColumnEvaluatorInfo.getFilterValues().getImplicitColumnFilterList()
.contains(shortBlockId)) {
isScanRequired = true;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d1d726a7/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index efc0e20..79f9b49 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -29,6 +29,7 @@ import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.core.scan.result.AbstractScannedResult;
import org.apache.carbondata.core.scan.result.impl.FilterQueryScannedResult;
@@ -107,9 +108,20 @@ public class FilterScanner extends AbstractBlockletScanner {
totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
// apply min max
if (isMinMaxEnabled) {
- BitSet bitSet = this.filterExecuter
- .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
- blocksChunkHolder.getDataBlock().getColumnsMinValue());
+ BitSet bitSet = null;
+ // check for implicit include filter instance
+ if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
+ String blockletId = blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR
+ + blocksChunkHolder.getDataBlock().blockletId();
+ bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
+ .isFilterValuesPresentInBlockOrBlocklet(
+ blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
+ blocksChunkHolder.getDataBlock().getColumnsMinValue(), blockletId);
+ } else {
+ bitSet = this.filterExecuter
+ .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
+ blocksChunkHolder.getDataBlock().getColumnsMinValue());
+ }
if (bitSet.isEmpty()) {
CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(),
blocksChunkHolder.getMeasureRawDataChunk());
[50/50] [abbrv] carbondata git commit: [CARBONDATA-1827] S3 Carbon
Implementation
Posted by ja...@apache.org.
[CARBONDATA-1827] S3 Carbon Implementation
1.Provide support for s3 in carbondata.
2.Added S3Example to create carbon table on s3.
3.Added S3CSVExample to load carbon table using csv from s3.
This closes #1805
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/00cc601d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/00cc601d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/00cc601d
Branch: refs/heads/carbonstore
Commit: 00cc601da33b9b3b4408a164f5cb6134a70c270f
Parents: d74e6e4
Author: SangeetaGulia <sa...@knoldus.in>
Authored: Thu Sep 21 14:56:26 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 13:21:26 2018 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 21 +++
.../filesystem/AbstractDFSCarbonFile.java | 20 ++-
.../datastore/filesystem/HDFSCarbonFile.java | 5 +-
.../core/datastore/impl/FileFactory.java | 11 +-
.../core/locks/CarbonLockFactory.java | 28 ++--
.../carbondata/core/locks/S3FileLock.java | 111 +++++++++++++
.../carbondata/core/util/CarbonProperties.java | 3 +-
.../filesystem/HDFSCarbonFileTest.java | 8 +-
examples/spark2/pom.xml | 5 +
examples/spark2/src/main/resources/data1.csv | 11 ++
.../carbondata/examples/S3CsvExample.scala | 99 +++++++++++
.../apache/carbondata/examples/S3Example.scala | 164 +++++++++++++++++++
.../spark/rdd/NewCarbonDataLoadRDD.scala | 42 ++++-
integration/spark2/pom.xml | 43 +++++
.../spark/rdd/CarbonDataRDDFactory.scala | 3 +-
.../org/apache/spark/sql/CarbonSession.scala | 59 +++++--
16 files changed, 595 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 77e8db8..4d838a8 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -167,6 +167,22 @@ public final class CarbonCommonConstants {
public static final String S3N_PREFIX = "s3n://";
public static final String S3A_PREFIX = "s3a://";
+ /**
+ * Access Key for s3n
+ */
+ public static final String S3N_ACCESS_KEY = "fs.s3n.awsAccessKeyId";
+ /**
+ * Secret Key for s3n
+ */
+ public static final String S3N_SECRET_KEY = "fs.s3n.awsSecretAccessKey";
+ /**
+ * Access Key for s3
+ */
+ public static final String S3_ACCESS_KEY = "fs.s3.awsAccessKeyId";
+ /**
+ * Secret Key for s3
+ */
+ public static final String S3_SECRET_KEY = "fs.s3.awsSecretAccessKey";
/**
* FS_DEFAULT_FS
@@ -941,6 +957,11 @@ public final class CarbonCommonConstants {
public static final String CARBON_LOCK_TYPE_HDFS = "HDFSLOCK";
/**
+ * S3LOCK TYPE
+ */
+ public static final String CARBON_LOCK_TYPE_S3 = "S3LOCK";
+
+ /**
* Invalid filter member log string
*/
public static final String FILTER_INVALID_MEMBER =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index 7b634d2..e1a34fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
-public abstract class AbstractDFSCarbonFile implements CarbonFile {
+public abstract class AbstractDFSCarbonFile implements CarbonFile {
/**
* LOGGER
*/
@@ -261,18 +261,28 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
@Override public DataOutputStream getDataOutputStream(String path, FileFactory.FileType fileType,
int bufferSize, boolean append) throws IOException {
Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
+ FileSystem fileSystem = pt.getFileSystem(FileFactory.getConfiguration());
FSDataOutputStream stream = null;
if (append) {
// append to a file only if file already exists else file not found
// exception will be thrown by hdfs
if (CarbonUtil.isFileExists(path)) {
- stream = fs.append(pt, bufferSize);
+ if (FileFactory.FileType.S3 == fileType) {
+ DataInputStream dataInputStream = fileSystem.open(pt);
+ int count = dataInputStream.available();
+ // create buffer
+ byte[] byteStreamBuffer = new byte[count];
+ dataInputStream.read(byteStreamBuffer);
+ stream = fileSystem.create(pt, true, bufferSize);
+ stream.write(byteStreamBuffer);
+ } else {
+ stream = fileSystem.append(pt, bufferSize);
+ }
} else {
- stream = fs.create(pt, true, bufferSize);
+ stream = fileSystem.create(pt, true, bufferSize);
}
} else {
- stream = fs.create(pt, true, bufferSize);
+ stream = fileSystem.create(pt, true, bufferSize);
}
return stream;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
index d470b47..892a556 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
@@ -107,8 +107,11 @@ public class HDFSCarbonFile extends AbstractDFSCarbonFile {
((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
org.apache.hadoop.fs.Options.Rename.OVERWRITE);
return true;
+ } else if (fileStatus.getPath().toString().startsWith("s3n")) {
+ fs.delete(new Path(changetoName), true);
+ return fs.rename(fileStatus.getPath(), new Path(changetoName));
} else {
- return false;
+ return fs.rename(fileStatus.getPath(), new Path(changetoName));
}
} catch (IOException e) {
LOGGER.error("Exception occured: " + e.getMessage());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index e6fbd04..daf6d93 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -246,7 +246,15 @@ public final class FileFactory {
*/
public static DataOutputStream getDataOutputStreamUsingAppend(String path, FileType fileType)
throws IOException {
- return getCarbonFile(path).getDataOutputStreamUsingAppend(path, fileType);
+ if (FileType.S3 == fileType) {
+ CarbonFile carbonFile = getCarbonFile(path);
+ if (carbonFile.exists()) {
+ carbonFile.delete();
+ }
+ return carbonFile.getDataOutputStream(path,fileType);
+ } else {
+ return getCarbonFile(path).getDataOutputStreamUsingAppend(path, fileType);
+ }
}
/**
@@ -423,6 +431,7 @@ public final class FileFactory {
throws IOException {
FileFactory.FileType fileType = FileFactory.getFileType(directoryPath);
switch (fileType) {
+ case S3:
case HDFS:
case VIEWFS:
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
index e70e655..3226a63 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
@@ -52,18 +52,21 @@ public class CarbonLockFactory {
*/
public static ICarbonLock getCarbonLockObj(AbsoluteTableIdentifier absoluteTableIdentifier,
String lockFile) {
- switch (lockTypeConfigured) {
- case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
- return new LocalFileLock(absoluteTableIdentifier, lockFile);
- case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
- return new ZooKeeperLocking(absoluteTableIdentifier, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
- return new HdfsFileLock(absoluteTableIdentifier, lockFile);
-
- default:
- throw new UnsupportedOperationException("Not supported the lock type");
+ String tablePath = absoluteTableIdentifier.getTablePath();
+ if (lockTypeConfigured.equals(CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER)) {
+ return new ZooKeeperLocking(absoluteTableIdentifier, lockFile);
+ } else if (tablePath.startsWith(CarbonCommonConstants.S3A_PREFIX) ||
+ tablePath.startsWith(CarbonCommonConstants.S3N_PREFIX) ||
+ tablePath.startsWith(CarbonCommonConstants.S3_PREFIX)) {
+ lockTypeConfigured = CarbonCommonConstants.CARBON_LOCK_TYPE_S3;
+ return new S3FileLock(absoluteTableIdentifier, lockFile);
+ } else if (tablePath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) {
+ lockTypeConfigured = CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS;
+ return new HdfsFileLock(absoluteTableIdentifier, lockFile);
+ } else {
+ lockTypeConfigured = CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL;
+ return new LocalFileLock(absoluteTableIdentifier, lockFile);
}
}
@@ -84,6 +87,9 @@ public class CarbonLockFactory {
case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
return new HdfsFileLock(locFileLocation, lockFile);
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_S3:
+ return new S3FileLock(locFileLocation, lockFile);
+
default:
throw new UnsupportedOperationException("Not supported the lock type");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/core/src/main/java/org/apache/carbondata/core/locks/S3FileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/S3FileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/S3FileLock.java
new file mode 100644
index 0000000..8836960
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/S3FileLock.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.locks;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+
+/**
+ * This class is used to handle the S3 File locking.
+ * This is acheived using the concept of acquiring the data out stream using Append option.
+ */
+public class S3FileLock extends AbstractCarbonLock {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(S3FileLock.class.getName());
+ /**
+ * location s3 file location
+ */
+ private String location;
+
+ private DataOutputStream dataOutputStream;
+
+ /**
+ * @param tableIdentifier
+ * @param lockFile
+ */
+ public S3FileLock(AbsoluteTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getTablePath(), lockFile);
+ }
+
+ /**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public S3FileLock(String lockFileLocation, String lockFile) {
+ this.location = lockFileLocation + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ LOGGER.info("S3 lock path:" + this.location);
+ initRetry();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.carbondata.core.locks.ICarbonLock#unlock()
+ */
+ @Override public boolean unlock() {
+ boolean status = false;
+ if (null != dataOutputStream) {
+ try {
+ dataOutputStream.close();
+ status = true;
+ } catch (IOException e) {
+ status = false;
+ } finally {
+ CarbonFile carbonFile =
+ FileFactory.getCarbonFile(location, FileFactory.getFileType(location));
+ if (carbonFile.exists()) {
+ if (carbonFile.delete()) {
+ LOGGER.info("Deleted the lock file " + location);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + location);
+ status = false;
+ }
+ } else {
+ LOGGER.error(
+ "Not able to delete the lock file because it is not existed in location " + location);
+ status = false;
+ }
+ }
+ }
+ return status;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.carbondata.core.locks.ICarbonLock#lock()
+ */
+ @Override public boolean lock() {
+ try {
+ if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
+ FileFactory.createNewLockFile(location, FileFactory.getFileType(location));
+ }
+ dataOutputStream =
+ FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location));
+ return true;
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return false;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index fd78efc..13c140f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -306,7 +306,8 @@ public final class CarbonProperties {
String defaultFs = configuration.get("fs.defaultFS");
if (null != defaultFs && (defaultFs.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)
|| defaultFs.startsWith(CarbonCommonConstants.VIEWFSURL_PREFIX) || defaultFs
- .startsWith(CarbonCommonConstants.ALLUXIOURL_PREFIX))
+ .startsWith(CarbonCommonConstants.ALLUXIOURL_PREFIX) || defaultFs
+ .startsWith(CarbonCommonConstants.S3A_PREFIX))
&& !CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS.equalsIgnoreCase(lockTypeConfigured)) {
LOGGER.warn("The value \"" + lockTypeConfigured + "\" configured for key "
+ LOCK_TYPE + " is invalid for current file system. "
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
index 7726693..4018123 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -369,7 +369,13 @@ public class HDFSCarbonFileTest {
}
};
- assertEquals(hdfsCarbonFile.renameForce(fileName), false);
+ new MockUp<WebHdfsFileSystem>(){
+ @Mock
+ public boolean rename(final Path src, final Path dst) throws IOException {
+ return true;
+ }
+ };
+ assertEquals(hdfsCarbonFile.renameForce(fileName), true);
}
@Test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/examples/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index da39f1d..b7f7498 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -62,6 +62,11 @@
<version>${spark.version}</version>
<scope>${spark.deps.scope}</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.2</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/examples/spark2/src/main/resources/data1.csv
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/resources/data1.csv b/examples/spark2/src/main/resources/data1.csv
new file mode 100644
index 0000000..cf732eb
--- /dev/null
+++ b/examples/spark2/src/main/resources/data1.csv
@@ -0,0 +1,11 @@
+shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField
+1,10,1100,48.4,spark,2015-4-23 12:01:01,1.23,2015-4-23,aaa,2.5
+5,17,1140,43.4,spark,2015-7-27 12:01:02,3.45,2015-7-27,bbb,2.5
+1,11,1100,44.4,flink,2015-5-23 12:01:03,23.23,2015-5-23,ccc,2.5
+1,10,1150,43.4,spark,2015-7-24 12:01:04,254.12,2015-7-24,ddd,2.5
+1,10,1100,47.4,spark,2015-7-23 12:01:05,876.14,2015-7-23,eeee,3.5
+3,14,1160,43.4,hive,2015-7-26 12:01:06,3454.32,2015-7-26,ff,2.5
+2,10,1100,43.4,impala,2015-7-23 12:01:07,456.98,2015-7-23,ggg,2.5
+1,10,1100,43.4,spark,2015-5-23 12:01:08,32.53,2015-5-23,hhh,2.5
+4,16,1130,42.4,impala,2015-7-23 12:01:09,67.23,2015-7-23,iii,2.5
+1,10,1100,43.4,spark,2015-7-23 12:01:10,832.23,2015-7-23,jjj,2.5
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3CsvExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3CsvExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3CsvExample.scala
new file mode 100644
index 0000000..b37fba8
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3CsvExample.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.hadoop.fs.s3a.Constants.{ACCESS_KEY, SECRET_KEY}
+import org.apache.spark.sql.SparkSession
+import org.slf4j.{Logger, LoggerFactory}
+
+object S3CsvExample {
+
+ /**
+ * This example demonstrate to create local store and load data from CSV files on S3
+ *
+ * @param args require three parameters "Access-key" "Secret-key"
+ * "s3 path to csv" "spark-master"
+ */
+ def main(args: Array[String]) {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val logger: Logger = LoggerFactory.getLogger(this.getClass)
+
+ import org.apache.spark.sql.CarbonSession._
+ if (args.length != 4) {
+ logger.error("Usage: java CarbonS3Example <access-key> <secret-key>" +
+ "<s3.csv.location> <spark-master>")
+ System.exit(0)
+ }
+
+ val spark = SparkSession
+ .builder()
+ .master(args(3))
+ .appName("S3CsvExample")
+ .config("spark.driver.host", "localhost")
+ .config("spark.hadoop." + ACCESS_KEY, args(0))
+ .config("spark.hadoop." + SECRET_KEY, args(1))
+ .getOrCreateCarbonSession()
+
+ spark.sparkContext.setLogLevel("INFO")
+
+ spark.sql(
+ s"""
+ | CREATE TABLE if not exists carbon_table1(
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT
+ | )
+ | STORED BY 'carbondata'
+ | LOCATION '$rootPath/examples/spark2/target/store'
+ | TBLPROPERTIES('SORT_COLUMNS'='', 'DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${ args(2) }'
+ | INTO TABLE carbon_table1
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '${ args(2) }'
+ | INTO TABLE carbon_table1
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table1
+ """.stripMargin).show()
+
+ spark.sql("Drop table if exists carbon_table1")
+
+ spark.stop()
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
new file mode 100644
index 0000000..d3d0a37
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3Example.scala
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.hadoop.fs.s3a.Constants.{ACCESS_KEY, ENDPOINT, SECRET_KEY}
+import org.apache.spark.sql.{Row, SparkSession}
+import org.slf4j.{Logger, LoggerFactory}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+
+object S3Example {
+
+ /**
+ * This example demonstrate usage of
+ * 1. create carbon table with storage location on object based storage
+ * like AWS S3, Huawei OBS, etc
+ * 2. load data into carbon table, the generated file will be stored on object based storage
+ * query the table.
+ *
+ * @param args require three parameters "Access-key" "Secret-key"
+ * "table-path on s3" "s3-endpoint" "spark-master"
+ */
+ def main(args: Array[String]) {
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val path = s"$rootPath/examples/spark2/src/main/resources/data1.csv"
+ val logger: Logger = LoggerFactory.getLogger(this.getClass)
+
+ import org.apache.spark.sql.CarbonSession._
+ if (args.length < 3 || args.length > 5) {
+ logger.error("Usage: java CarbonS3Example <access-key> <secret-key>" +
+ "<table-path-on-s3> [s3-endpoint] [spark-master]")
+ System.exit(0)
+ }
+
+ val (accessKey, secretKey, endpoint) = getKeyOnPrefix(args(2))
+ val spark = SparkSession
+ .builder()
+ .master(getSparkMaster(args))
+ .appName("S3Example")
+ .config("spark.driver.host", "localhost")
+ .config(accessKey, args(0))
+ .config(secretKey, args(1))
+ .config(endpoint, getS3EndPoint(args))
+ .getOrCreateCarbonSession()
+
+ spark.sparkContext.setLogLevel("WARN")
+
+ spark.sql("Drop table if exists carbon_table")
+
+ spark.sql(
+ s"""
+ | CREATE TABLE if not exists carbon_table(
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT
+ | )
+ | STORED BY 'carbondata'
+ | LOCATION '${ args(2) }'
+ | TBLPROPERTIES('SORT_COLUMNS'='', 'DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon_table
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ """.stripMargin).show()
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon_table
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE carbon_table
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ val countSegment: Array[Row] =
+ spark.sql(
+ s"""
+ | SHOW SEGMENTS FOR TABLE carbon_table
+ """.stripMargin).collect()
+
+ while (countSegment.length != 3) {
+ this.wait(2000)
+ }
+
+ // Use compaction command to merge segments or small files in object based storage,
+ // this can be done periodically.
+ spark.sql("ALTER table carbon_table compact 'MAJOR'")
+ spark.sql("show segments for table carbon_table").show()
+
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ """.stripMargin).show()
+
+ spark.sql("Drop table if exists carbon_table")
+
+ spark.stop()
+ }
+
+ def getKeyOnPrefix(path: String): (String, String, String) = {
+ val endPoint = "spark.hadoop." + ENDPOINT
+ if (path.startsWith(CarbonCommonConstants.S3A_PREFIX)) {
+ ("spark.hadoop." + ACCESS_KEY, "spark.hadoop." + SECRET_KEY, endPoint)
+ } else if (path.startsWith(CarbonCommonConstants.S3N_PREFIX)) {
+ ("spark.hadoop." + CarbonCommonConstants.S3N_ACCESS_KEY,
+ "spark.hadoop." + CarbonCommonConstants.S3N_SECRET_KEY, endPoint)
+ } else if (path.startsWith(CarbonCommonConstants.S3_PREFIX)) {
+ ("spark.hadoop." + CarbonCommonConstants.S3_ACCESS_KEY,
+ "spark.hadoop." + CarbonCommonConstants.S3_SECRET_KEY, endPoint)
+ } else {
+ throw new Exception("Incorrect Store Path")
+ }
+ }
+
+ def getS3EndPoint(args: Array[String]): String = {
+ if (args.length >= 4 && args(3).contains(".com")) args(3)
+ else ""
+ }
+
+ def getSparkMaster(args: Array[String]): String = {
+ if (args.length == 5) args(4)
+ else if (args(3).contains("spark:") || args(3).contains("mesos:")) args(3)
+ else "local"
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 1fa1689..917fc88 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -41,6 +41,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.compression.CompressorFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -346,11 +347,31 @@ class NewDataFrameLoaderRDD[K, V](
sc: SparkContext,
result: DataLoadResult[K, V],
carbonLoadModel: CarbonLoadModel,
- prev: DataLoadCoalescedRDD[Row]) extends CarbonRDD[(K, V)](prev) {
+ prev: DataLoadCoalescedRDD[Row],
+ @transient hadoopConf: Configuration) extends CarbonRDD[(K, V)](prev) {
- override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
+ private val confBytes = {
+ val bao = new ByteArrayOutputStream()
+ val oos = new ObjectOutputStream(bao)
+ hadoopConf.write(oos)
+ oos.close()
+ CompressorFactory.getInstance().getCompressor.compressByte(bao.toByteArray)
+ }
+ private def getConf = {
+ val configuration = new Configuration(false)
+ val bai = new ByteArrayInputStream(CompressorFactory.getInstance().getCompressor
+ .unCompressByte(confBytes))
+ val ois = new ObjectInputStream(bai)
+ configuration.readFields(ois)
+ ois.close()
+ configuration
+ }
+
+ override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+ val hadoopConf = getConf
+ setS3Configurations(hadoopConf)
val iter = new Iterator[(K, V)] {
val loadMetadataDetails = new LoadMetadataDetails()
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
@@ -420,6 +441,23 @@ class NewDataFrameLoaderRDD[K, V](
iter
}
override protected def getPartitions: Array[Partition] = firstParent[Row].partitions
+
+ private def setS3Configurations(hadoopConf: Configuration): Unit = {
+ FileFactory.getConfiguration
+ .set("fs.s3a.access.key", hadoopConf.get("fs.s3a.access.key", ""))
+ FileFactory.getConfiguration
+ .set("fs.s3a.secret.key", hadoopConf.get("fs.s3a.secret.key", ""))
+ FileFactory.getConfiguration
+ .set("fs.s3a.endpoint", hadoopConf.get("fs.s3a.endpoint", ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3_ACCESS_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3_ACCESS_KEY, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3_SECRET_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3_SECRET_KEY, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3N_ACCESS_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3N_ACCESS_KEY, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3N_SECRET_KEY,
+ hadoopConf.get(CarbonCommonConstants.S3N_SECRET_KEY, ""))
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index 60cb61f..4a37abc 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -57,6 +57,49 @@
<version>2.2.1</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.7.4</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ <version>0.9.0</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 6f056a6..655e5a0 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -992,7 +992,8 @@ object CarbonDataRDDFactory {
sqlContext.sparkContext,
new DataLoadResultImpl(),
carbonLoadModel,
- newRdd
+ newRdd,
+ sqlContext.sparkContext.hadoopConfiguration
).collect()
} catch {
case ex: Exception =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00cc601d/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index e95b8db..b485546 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -21,16 +21,21 @@ import java.io.File
import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.s3a.Constants.{ACCESS_KEY, ENDPOINT, SECRET_KEY}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
import org.apache.spark.sql.SparkSession.Builder
+import org.apache.spark.sql.execution.command.preaaggregate._
import org.apache.spark.sql.execution.streaming.CarbonStreamingQueryListener
import org.apache.spark.sql.hive.execution.command.CarbonSetCommand
import org.apache.spark.sql.internal.{SessionState, SharedState}
import org.apache.spark.util.{CarbonReflectionUtils, Utils}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, ThreadLocalSessionInfo}
+import org.apache.carbondata.events._
+import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePreExecutionEvent, LoadTablePreStatusUpdateEvent}
/**
* Session implementation for {org.apache.spark.sql.SparkSession}
@@ -71,6 +76,10 @@ class CarbonSession(@transient val sc: SparkContext,
new CarbonSession(sparkContext, Some(sharedState))
}
+ if (existingSharedState.isEmpty) {
+ CarbonSession.initListeners()
+ }
+
}
object CarbonSession {
@@ -152,6 +161,7 @@ object CarbonSession {
sparkConf.setAppName(randomAppName)
}
val sc = SparkContext.getOrCreate(sparkConf)
+ setS3Configurations(sc)
// maybe this is an existing SparkContext, update its SparkConf which maybe used
// by SparkSession
options.foreach { case (k, v) => sc.conf.set(k, v) }
@@ -214,31 +224,17 @@ object CarbonSession {
ThreadLocalSessionInfo.setCarbonSessionInfo(currentThreadSessionInfo)
}
-
- def threadSet(key: String, value: Object): Unit = {
- var currentThreadSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
- if (currentThreadSessionInfo == null) {
- currentThreadSessionInfo = new CarbonSessionInfo()
- }
- else {
- currentThreadSessionInfo = currentThreadSessionInfo.clone()
- }
- currentThreadSessionInfo.getThreadParams.setExtraInfo(key, value)
- ThreadLocalSessionInfo.setCarbonSessionInfo(currentThreadSessionInfo)
- }
-
def threadUnset(key: String): Unit = {
val currentThreadSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
if (currentThreadSessionInfo != null) {
val currentThreadSessionInfoClone = currentThreadSessionInfo.clone()
val threadParams = currentThreadSessionInfoClone.getThreadParams
CarbonSetCommand.unsetValue(threadParams, key)
- threadParams.removeExtraInfo(key)
ThreadLocalSessionInfo.setCarbonSessionInfo(currentThreadSessionInfoClone)
}
}
- def updateSessionInfoToCurrentThread(sparkSession: SparkSession): Unit = {
+ private[spark] def updateSessionInfoToCurrentThread(sparkSession: SparkSession): Unit = {
val carbonSessionInfo = CarbonEnv.getInstance(sparkSession).carbonSessionInfo.clone()
val currentThreadSessionInfoOrig = ThreadLocalSessionInfo.getCarbonSessionInfo
if (currentThreadSessionInfoOrig != null) {
@@ -252,4 +248,37 @@ object CarbonSession {
ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)
}
+ def initListeners(): Unit = {
+ OperationListenerBus.getInstance()
+ .addListener(classOf[LoadTablePreStatusUpdateEvent], LoadPostAggregateListener)
+ .addListener(classOf[DeleteSegmentByIdPreEvent], PreAggregateDeleteSegmentByIdPreListener)
+ .addListener(classOf[DeleteSegmentByDatePreEvent], PreAggregateDeleteSegmentByDatePreListener)
+ .addListener(classOf[UpdateTablePreEvent], UpdatePreAggregatePreListener)
+ .addListener(classOf[DeleteFromTablePreEvent], DeletePreAggregatePreListener)
+ .addListener(classOf[DeleteFromTablePreEvent], DeletePreAggregatePreListener)
+ .addListener(classOf[AlterTableDropColumnPreEvent], PreAggregateDropColumnPreListener)
+ .addListener(classOf[AlterTableRenamePreEvent], PreAggregateRenameTablePreListener)
+ .addListener(classOf[AlterTableDataTypeChangePreEvent], PreAggregateDataTypeChangePreListener)
+ .addListener(classOf[AlterTableAddColumnPreEvent], PreAggregateAddColumnsPreListener)
+ .addListener(classOf[LoadTablePreExecutionEvent], LoadPreAggregateTablePreListener)
+ .addListener(classOf[AlterTableCompactionPreStatusUpdateEvent],
+ AlterPreAggregateTableCompactionPostListener)
+ }
+
+ private def setS3Configurations(sparkContext: SparkContext): Unit = {
+ FileFactory.getConfiguration
+ .set(ACCESS_KEY, sparkContext.hadoopConfiguration.get(ACCESS_KEY, ""))
+ FileFactory.getConfiguration
+ .set(SECRET_KEY, sparkContext.hadoopConfiguration.get(SECRET_KEY, ""))
+ FileFactory.getConfiguration
+ .set(ENDPOINT, sparkContext.hadoopConfiguration.get(ENDPOINT, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3_ACCESS_KEY,
+ sparkContext.hadoopConfiguration.get(CarbonCommonConstants.S3_ACCESS_KEY, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3_SECRET_KEY,
+ sparkContext.hadoopConfiguration.get(CarbonCommonConstants.S3_SECRET_KEY, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3N_ACCESS_KEY,
+ sparkContext.hadoopConfiguration.get(CarbonCommonConstants.S3N_ACCESS_KEY, ""))
+ FileFactory.getConfiguration.set(CarbonCommonConstants.S3N_SECRET_KEY,
+ sparkContext.hadoopConfiguration.get(CarbonCommonConstants.S3N_SECRET_KEY, ""))
+ }
}
[32/50] [abbrv] carbondata git commit: [CARBONDATA-2016] Exception
displays while executing compaction with alter query
Posted by ja...@apache.org.
[CARBONDATA-2016] Exception displays while executing compaction with alter query
Reason:
When we apply the alter table command to add column with default value it is always storing it as long object for all measures,it is wrongly written
in restructure util we should return the value as the same type as that of the measure,it was causing the compaction to fail with class cast exception
because the data type and its corresponding value does not have same data type
Solution: Correct the wrong logic in restructure util the type of returning value object should be same as that of measure
This closes #1839
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a597c2f9
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a597c2f9
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a597c2f9
Branch: refs/heads/carbonstore
Commit: a597c2f9ba5213afd17fec223b5b5a4609c564bf
Parents: 12ccf70
Author: anubhav100 <an...@knoldus.in>
Authored: Fri Jan 19 21:31:14 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Mon Jan 29 14:14:03 2018 +0530
----------------------------------------------------------------------
.../scan/executor/util/RestructureUtil.java | 13 +++++---
.../iud/HorizontalCompactionTestCase.scala | 32 ++++++++++++++++++++
2 files changed, 41 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a597c2f9/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index 572400d..288aea7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -178,8 +178,7 @@ public class RestructureUtil {
if (isDefaultValueNull(defaultValue)) {
dictionaryDefaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
} else {
- dictionaryDefaultValue =
- CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1;
+ dictionaryDefaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1;
}
return dictionaryDefaultValue;
}
@@ -285,11 +284,17 @@ public class RestructureUtil {
public static Object getMeasureDefaultValue(ColumnSchema columnSchema, byte[] defaultValue) {
Object measureDefaultValue = null;
if (!isDefaultValueNull(defaultValue)) {
- String value = null;
+ String value;
DataType dataType = columnSchema.getDataType();
- if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) {
+ if (dataType == DataTypes.SHORT) {
+ value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ measureDefaultValue = Short.valueOf(value);
+ } else if (dataType == DataTypes.LONG) {
value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
measureDefaultValue = Long.parseLong(value);
+ } else if (dataType == DataTypes.INT) {
+ value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ measureDefaultValue = Integer.parseInt(value);
} else if (DataTypes.isDecimal(dataType)) {
BigDecimal decimal = DataTypeUtil.byteToBigDecimal(defaultValue);
if (columnSchema.getScale() > decimal.scale()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a597c2f9/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala
index 9d4b708..af1846b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala
@@ -350,8 +350,39 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
)
sql("""drop table dest2""")
}
+ test("test the compaction after alter command") // As per bug Carbondata-2016
+ {
+ sql(
+ "CREATE TABLE CUSTOMER1 ( C_CUSTKEY INT , C_NAME STRING , C_ADDRESS STRING , C_NATIONKEY INT , C_PHONE STRING , C_ACCTBAL DECIMAL(15,2) , C_MKTSEGMENT STRING , C_COMMENT STRING) stored by 'carbondata'")
+ sql(
+ "insert into customer1 values(1,'vandana','noida',1,'123456789',45987.78,'hello','comment')")
+ sql(
+ "insert into customer1 values(2,'vandana','noida',2,'123456789',487.78,'hello','comment')")
+
+ sql(
+ " insert into customer1 values(3,'geetika','delhi',3,'123456789',487897.78,'hello','comment')")
+
+ sql(
+ "insert into customer1 values(4,'sangeeta','delhi',3,'123456789',48789.78,'hello','comment')")
+
+ sql(
+ "alter table customer1 add columns (shortfield short) TBLPROPERTIES ('DEFAULT.VALUE.shortfield'='32767')")
+
+ sql(
+ "alter table customer1 add columns (intfield int) TBLPROPERTIES ('DEFAULT.VALUE.intfield'='2147483647')")
+
+ sql(
+ "alter table customer1 add columns (longfield bigint) TBLPROPERTIES ('DEFAULT.VALUE.longfield'='9223372036854775807')")
+
+ sql("alter table customer1 compact 'minor' ").show()
+
+ checkAnswer(sql("select shortfield from customer1"),Seq(Row(32767),Row(32767),Row(32767),Row(32767)))
+ checkAnswer(sql("select intfield from customer1"),Seq(Row(2147483647),Row(2147483647),Row(2147483647),Row(2147483647)))
+ checkAnswer(sql("select longfield from customer1"),Seq(Row(9223372036854775807L),Row(9223372036854775807L),Row(9223372036854775807L),Row(9223372036854775807L)))
+
+ }
override def afterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_VECTOR_READER , "true")
@@ -360,6 +391,7 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.isHorizontalCompactionEnabled , "true")
sql("""drop table if exists t_carbn01""")
+ sql("""drop table if exists customer1""")
}
}
[08/50] [abbrv] carbondata git commit: [CARBONDATA-2058] Block append
data to streaming segment after writing exception
Posted by ja...@apache.org.
[CARBONDATA-2058] Block append data to streaming segment after writing exception
If CarbonStreamRecordWriter happened an exception, mark hasException attribute to avoid appending data to the segment.
This closes #1836
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bcc9cf0e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bcc9cf0e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bcc9cf0e
Branch: refs/heads/carbonstore
Commit: bcc9cf0e99bf93472572863234604e5eaf515a76
Parents: 9882a74
Author: QiangCai <qi...@qq.com>
Authored: Fri Jan 19 17:28:57 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Jan 19 21:06:45 2018 +0800
----------------------------------------------------------------------
.../carbondata/hadoop/streaming/CarbonStreamRecordWriter.java | 4 ++++
.../org/apache/carbondata/streaming/segment/StreamSegment.java | 1 +
2 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bcc9cf0e/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
index 7d862d4..212baa7 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
@@ -319,4 +319,8 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
public String getFileName() {
return fileName;
}
+
+ public void setHasException(boolean hasException) {
+ this.hasException = hasException;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bcc9cf0e/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
index 69d0b8d..74e1461 100644
--- a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
+++ b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
@@ -248,6 +248,7 @@ public class StreamSegment {
if (writer != null) {
LOGGER.error(ex, "Failed to append batch data to stream segment: " +
writer.getSegmentDir());
+ writer.setHasException(true);
}
throw ex;
} finally {
[45/50] [abbrv] carbondata git commit: [CARBONDATA-2090] Fix the
error message of alter streaming property
Posted by ja...@apache.org.
[CARBONDATA-2090] Fix the error message of alter streaming property
Fix the error message of alter streaming property
This closes #1873
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c9a501dc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c9a501dc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c9a501dc
Branch: refs/heads/carbonstore
Commit: c9a501dca672678093fe0e5f526b226057b96d5d
Parents: 4d3f398
Author: QiangCai <qi...@qq.com>
Authored: Mon Jan 29 11:40:47 2018 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jan 31 11:58:44 2018 +0800
----------------------------------------------------------------------
.../spark/sql/execution/strategy/DDLStrategy.scala | 7 ++++++-
.../spark/carbondata/TestStreamingTableOperation.scala | 11 ++++++++++-
2 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9a501dc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 57be754..db8c6a2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -231,9 +231,14 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
// TODO remove this limitation later
val property = properties.find(_._1.equalsIgnoreCase("streaming"))
if (property.isDefined) {
- if (!property.get._2.trim.equalsIgnoreCase("true")) {
+ if (carbonTable.isStreamingTable) {
throw new MalformedCarbonCommandException(
"Streaming property can not be changed once it is 'true'")
+ } else {
+ if (!property.get._2.trim.equalsIgnoreCase("true")) {
+ throw new MalformedCarbonCommandException(
+ "Streaming property value is incorrect")
+ }
}
}
ExecutedCommandExec(CarbonAlterTableSetCommand(tableName, properties, isView)) :: Nil
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c9a501dc/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 18e52f6..44204d4 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -181,7 +181,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
}
// normal table not support streaming ingest
- test("normal table not support streaming ingest") {
+ test("normal table not support streaming ingest and alter normal table's streaming property") {
+ // alter normal table's streaming property
+ val msg = intercept[MalformedCarbonCommandException](sql("alter table streaming.batch_table set tblproperties('streaming'='false')"))
+ assertResult("Streaming property value is incorrect")(msg.getMessage)
+
val identifier = new TableIdentifier("batch_table", Option("streaming"))
val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.lookupRelation(identifier)(spark)
.asInstanceOf[CarbonRelation].metaData.carbonTable
@@ -518,6 +522,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
case _ =>
assert(false, "should support set table to streaming")
}
+
+ // alter streaming table's streaming property
+ val msg = intercept[MalformedCarbonCommandException](sql("alter table streaming.stream_table_handoff set tblproperties('streaming'='false')"))
+ assertResult("Streaming property can not be changed once it is 'true'")(msg.getMessage)
+
val segments = sql("show segments for table streaming.stream_table_handoff").collect()
assert(segments.length == 2 || segments.length == 3)
assertResult("Streaming")(segments(0).getString(1))
[17/50] [abbrv] carbondata git commit: [CARBONDATA-2067] Fix NPE
exception in StreamHandoffRDD
Posted by ja...@apache.org.
[CARBONDATA-2067] Fix NPE exception in StreamHandoffRDD
To avoid NPE of carbon table in StreamHandoffRDD, add carbon table to cache in internalCompute method
This closes #1846
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3312ee72
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3312ee72
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3312ee72
Branch: refs/heads/carbonstore
Commit: 3312ee72c5e67a3e7f00a47f925b6832a00b14a6
Parents: ee11bb1
Author: QiangCai <qi...@qq.com>
Authored: Tue Jan 23 16:20:19 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Jan 25 15:46:04 2018 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/streaming/StreamHandoffRDD.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3312ee72/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index b91be0c..d092580 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -17,7 +17,6 @@
package org.apache.carbondata.streaming
-import java.io.IOException
import java.text.SimpleDateFormat
import java.util
import java.util.Date
@@ -32,7 +31,8 @@ import org.apache.spark.sql.SparkSession
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.datastore.block.SegmentProperties
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
+import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
+import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus, SegmentStatusManager}
@@ -46,7 +46,6 @@ import org.apache.carbondata.processing.merger.{CompactionResultSortProcessor, C
import org.apache.carbondata.processing.util.CarbonLoaderUtil
import org.apache.carbondata.spark.{HandoffResult, HandoffResultImpl}
import org.apache.carbondata.spark.rdd.CarbonRDD
-import org.apache.carbondata.streaming.segment.StreamSegment
/**
* partition of the handoff segment
@@ -106,6 +105,7 @@ class StreamHandoffRDD[K, V](
carbonLoadModel.setPartitionId("0")
carbonLoadModel.setTaskNo("" + split.index)
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ CarbonMetadata.getInstance().addCarbonTable(carbonTable)
// the input iterator is using raw row
val iteratorList = prepareInputIterator(split, carbonTable)
// use CompactionResultSortProcessor to sort data dan write to columnar files
[20/50] [abbrv] carbondata git commit: [CARBONDATA-2076] Refactored
code segregated process meta and process data in load command
Posted by ja...@apache.org.
[CARBONDATA-2076] Refactored code segregated process meta and process data in load command
This closes #1837
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3a6136df
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3a6136df
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3a6136df
Branch: refs/heads/carbonstore
Commit: 3a6136df066b9d0cdeecec283115e4a99d82a900
Parents: 7d43442
Author: kumarvishal <ku...@gmail.com>
Authored: Fri Jan 19 17:22:28 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Jan 25 16:28:34 2018 +0530
----------------------------------------------------------------------
.../indexstore/BlockletDataMapIndexStore.java | 2 +-
.../carbondata/core/util/SessionParams.java | 14 +-
.../hadoop/api/CarbonOutputCommitter.java | 28 ++-
.../spark/rdd/AggregateDataMapCompactor.scala | 39 ++--
.../spark/rdd/CarbonDataRDDFactory.scala | 6 +-
.../spark/rdd/CarbonTableCompactor.scala | 4 +-
.../spark/rdd/CompactionFactory.scala | 10 +-
.../scala/org/apache/spark/sql/CarbonEnv.scala | 4 +-
.../org/apache/spark/sql/CarbonSession.scala | 18 +-
.../CarbonAlterTableCompactionCommand.scala | 53 ++---
.../management/CarbonInsertIntoCommand.scala | 24 ++-
.../management/CarbonLoadDataCommand.scala | 111 ++++-------
.../CreatePreAggregateTableCommand.scala | 66 ++++---
.../preaaggregate/PreAggregateListeners.scala | 194 +++++++++++++++----
.../preaaggregate/PreAggregateUtil.scala | 76 ++++----
.../CarbonCreateTableAsSelectCommand.scala | 27 +--
.../datasources/CarbonFileFormat.scala | 4 -
.../sql/execution/strategy/DDLStrategy.scala | 2 +-
.../apache/spark/sql/hive/CarbonRelation.scala | 16 +-
.../processing/loading/events/LoadEvents.java | 19 ++
20 files changed, 447 insertions(+), 270 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index ad80fd7..111a7a2 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -138,7 +138,7 @@ public class BlockletDataMapIndexStore
partitionFileStore.readAllPartitionsOfSegment(carbonFiles, segmentPath);
partitionFileStoreMap.put(identifier.getSegmentId(), partitionFileStore);
for (CarbonFile file : carbonFiles) {
- blockMetaInfoMap.put(file.getAbsolutePath(),
+ blockMetaInfoMap.put(FileFactory.getUpdatedFilePath(file.getAbsolutePath()),
new BlockMetaInfo(file.getLocations(), file.getSize()));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
index 3f0e856..afbd947 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java
@@ -52,10 +52,19 @@ public class SessionParams implements Serializable {
private Map<String, String> sProps;
private Map<String, String> addedProps;
-
+ private Map<String, Object> extraInfo;
public SessionParams() {
sProps = new HashMap<>();
addedProps = new HashMap<>();
+ extraInfo = new HashMap<>();
+ }
+
+ public void setExtraInfo(String key, Object value) {
+ this.extraInfo.put(key, value);
+ }
+
+ public Object getExtraInfo(String key) {
+ return this.extraInfo.get(key);
}
/**
@@ -198,6 +207,9 @@ public class SessionParams implements Serializable {
sProps.remove(property);
}
+ public void removeExtraInfo(String key) {
+ extraInfo.remove(key);
+ }
/**
* clear the set properties
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index eb18bbd..f6e928d 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -18,7 +18,10 @@
package org.apache.carbondata.hadoop.api;
import java.io.IOException;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -30,11 +33,12 @@ import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonSessionInfo;
+import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter;
import org.apache.carbondata.events.OperationContext;
import org.apache.carbondata.events.OperationListenerBus;
-import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
import org.apache.carbondata.processing.loading.events.LoadEvents;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.util.CarbonLoaderUtil;
@@ -106,18 +110,13 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
long segmentSize = CarbonLoaderUtil
.addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(), carbonTable);
if (segmentSize > 0 || overwriteSet) {
- String operationContextStr =
- context.getConfiguration().get(
- CarbonTableOutputFormat.OPERATION_CONTEXT,
- null);
- if (operationContextStr != null) {
- OperationContext operationContext =
- (OperationContext) ObjectSerializationUtil.convertStringToObject(operationContextStr);
+ Object operationContext = getOperationContext();
+ if (operationContext != null) {
LoadEvents.LoadTablePreStatusUpdateEvent event =
new LoadEvents.LoadTablePreStatusUpdateEvent(carbonTable.getCarbonTableIdentifier(),
loadModel);
try {
- OperationListenerBus.getInstance().fireEvent(event, operationContext);
+ OperationListenerBus.getInstance().fireEvent(event, (OperationContext) operationContext);
} catch (Exception e) {
throw new IOException(e);
}
@@ -145,6 +144,15 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
}
}
+ private Object getOperationContext() {
+ // when validate segments is disabled in thread local update it to CarbonTableInputFormat
+ CarbonSessionInfo carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo();
+ if (carbonSessionInfo != null) {
+ return carbonSessionInfo.getThreadParams().getExtraInfo("partition.operationcontext");
+ }
+ return null;
+ }
+
/**
* Merge index files to a new single file.
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
index 636d731..5f8f389 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/AggregateDataMapCompactor.scala
@@ -24,11 +24,11 @@ import org.apache.spark.sql.{CarbonSession, SQLContext}
import org.apache.spark.sql.execution.command.CompactionModel
import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
-import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.events.OperationContext
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CompactionType}
@@ -39,7 +39,8 @@ class AggregateDataMapCompactor(carbonLoadModel: CarbonLoadModel,
compactionModel: CompactionModel,
executor: ExecutorService,
sqlContext: SQLContext,
- storeLocation: String)
+ storeLocation: String,
+ operationContext: OperationContext)
extends Compactor(carbonLoadModel, compactionModel, executor, sqlContext, storeLocation) {
override def executeCompaction(): Unit = {
@@ -57,30 +58,17 @@ class AggregateDataMapCompactor(carbonLoadModel: CarbonLoadModel,
CarbonCommonConstants.VALIDATE_CARBON_INPUT_SEGMENTS +
carbonLoadModel.getDatabaseName + "." +
carbonLoadModel.getTableName, "false")
- val headers = carbonTable.getTableInfo.getFactTable.getListOfColumns.asScala
- .map(_.getColumnName).mkString(",")
- // Creating a new query string to insert data into pre-aggregate table from that same table.
- // For example: To compact preaggtable1 we can fire a query like insert into preaggtable1
- // select * from preaggtable1
- // The following code will generate the select query with a load UDF that will be used to
- // apply DataLoadingRules
- val childDataFrame = sqlContext.sparkSession.sql(new CarbonSpark2SqlParser()
- // adding the aggregation load UDF
- .addPreAggLoadFunction(
- // creating the select query on the bases on table schema
- PreAggregateUtil.createChildSelectQuery(
- carbonTable.getTableInfo.getFactTable, carbonTable.getDatabaseName))).drop("preAggLoad")
+ CarbonSession.updateSessionInfoToCurrentThread(sqlContext.sparkSession)
+ val loadCommand = operationContext.getProperty(carbonTable.getTableName + "_Compaction")
+ .asInstanceOf[CarbonLoadDataCommand]
try {
- CarbonLoadDataCommand(
- Some(carbonTable.getDatabaseName),
- carbonTable.getTableName,
- null,
- Nil,
- Map("fileheader" -> headers),
- isOverwriteTable = false,
- dataFrame = Some(childDataFrame),
- internalOptions = Map(CarbonCommonConstants.IS_INTERNAL_LOAD_CALL -> "true",
- "mergedSegmentName" -> mergedLoadName)).run(sqlContext.sparkSession)
+ val newInternalOptions = loadCommand.internalOptions ++
+ Map("mergedSegmentName" -> mergedLoadName)
+ loadCommand.internalOptions = newInternalOptions
+ loadCommand.dataFrame =
+ Some(PreAggregateUtil.getDataFrame(
+ sqlContext.sparkSession, loadCommand.logicalPlan.get))
+ loadCommand.processData(sqlContext.sparkSession)
val newLoadMetaDataDetails = SegmentStatusManager.readLoadMetadata(
carbonTable.getMetaDataFilepath)
val updatedLoadMetaDataDetails = newLoadMetaDataDetails collect {
@@ -103,6 +91,7 @@ class AggregateDataMapCompactor(carbonLoadModel: CarbonLoadModel,
// For example: after 8.1 creation 0.1, 4.1, 8.1 have to be merged to 0.2 if threshhold
// allows it.
if (!compactionModel.compactionType.equals(CompactionType.MAJOR)) {
+
executeCompaction()
}
CarbonSession
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index f37fbd7..809c8ff 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -173,7 +173,8 @@ object CarbonDataRDDFactory {
compactionModel,
executor,
sqlContext,
- storeLocation)
+ storeLocation,
+ operationContext)
try {
// compaction status of the table which is triggered by the user.
var triggeredCompactionStatus = false
@@ -225,7 +226,8 @@ object CarbonDataRDDFactory {
newcompactionModel,
executor,
sqlContext,
- storeLocation).executeCompaction()
+ storeLocation,
+ operationContext).executeCompaction()
} catch {
case e: Exception =>
LOGGER.error("Exception in compaction thread for table " +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index 0dc856d..a0c8f65 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -45,7 +45,8 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
compactionModel: CompactionModel,
executor: ExecutorService,
sqlContext: SQLContext,
- storeLocation: String)
+ storeLocation: String,
+ operationContext: OperationContext)
extends Compactor(carbonLoadModel, compactionModel, executor, sqlContext, storeLocation) {
override def executeCompaction(): Unit = {
@@ -170,7 +171,6 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
carbonLoadModel.setLoadMetadataDetails(
SegmentStatusManager.readLoadMetadata(carbonTable.getMetaDataFilepath).toList.asJava)
// trigger event for compaction
- val operationContext = new OperationContext
val alterTableCompactionPreEvent: AlterTableCompactionPreEvent =
AlterTableCompactionPreEvent(sqlContext.sparkSession,
carbonTable,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CompactionFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CompactionFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CompactionFactory.scala
index 6060f06..8508d2a 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CompactionFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CompactionFactory.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.ExecutorService
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.execution.command.CompactionModel
+import org.apache.carbondata.events.OperationContext
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
object CompactionFactory {
@@ -33,21 +34,24 @@ object CompactionFactory {
compactionModel: CompactionModel,
executor: ExecutorService,
sqlContext: SQLContext,
- storeLocation: String): Compactor = {
+ storeLocation: String,
+ operationContext: OperationContext): Compactor = {
if (carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.isChildDataMap) {
new AggregateDataMapCompactor(
carbonLoadModel,
compactionModel,
executor,
sqlContext,
- storeLocation)
+ storeLocation,
+ operationContext)
} else {
new CarbonTableCompactor(
carbonLoadModel,
compactionModel,
executor,
sqlContext,
- storeLocation)
+ storeLocation,
+ operationContext)
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index bbc3c2d..585fe67 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -32,7 +32,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util._
import org.apache.carbondata.events._
-import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePreExecutionEvent, LoadTablePreStatusUpdateEvent}
+import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadMetadataEvent, LoadTablePreExecutionEvent, LoadTablePreStatusUpdateEvent}
import org.apache.carbondata.spark.rdd.SparkReadSupport
import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
@@ -143,6 +143,8 @@ object CarbonEnv {
.addListener(classOf[LoadTablePreExecutionEvent], LoadPreAggregateTablePreListener)
.addListener(classOf[AlterTableCompactionPreStatusUpdateEvent],
AlterPreAggregateTableCompactionPostListener)
+ .addListener(classOf[LoadMetadataEvent], LoadProcessMetaListener)
+ .addListener(classOf[LoadMetadataEvent], CompactionProcessMetaListener)
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index c2c15fe..e95b8db 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -53,7 +53,7 @@ class CarbonSession(@transient val sc: SparkContext,
* and a catalog that interacts with external systems.
*/
@transient
- override lazy val sharedState: SharedState = {
+ override lazy val sharedState: SharedState = {
existingSharedState match {
case Some(_) =>
val ss = existingSharedState.get
@@ -214,17 +214,31 @@ object CarbonSession {
ThreadLocalSessionInfo.setCarbonSessionInfo(currentThreadSessionInfo)
}
+
+ def threadSet(key: String, value: Object): Unit = {
+ var currentThreadSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
+ if (currentThreadSessionInfo == null) {
+ currentThreadSessionInfo = new CarbonSessionInfo()
+ }
+ else {
+ currentThreadSessionInfo = currentThreadSessionInfo.clone()
+ }
+ currentThreadSessionInfo.getThreadParams.setExtraInfo(key, value)
+ ThreadLocalSessionInfo.setCarbonSessionInfo(currentThreadSessionInfo)
+ }
+
def threadUnset(key: String): Unit = {
val currentThreadSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
if (currentThreadSessionInfo != null) {
val currentThreadSessionInfoClone = currentThreadSessionInfo.clone()
val threadParams = currentThreadSessionInfoClone.getThreadParams
CarbonSetCommand.unsetValue(threadParams, key)
+ threadParams.removeExtraInfo(key)
ThreadLocalSessionInfo.setCarbonSessionInfo(currentThreadSessionInfoClone)
}
}
- private[spark] def updateSessionInfoToCurrentThread(sparkSession: SparkSession): Unit = {
+ def updateSessionInfoToCurrentThread(sparkSession: SparkSession): Unit = {
val carbonSessionInfo = CarbonEnv.getInstance(sparkSession).carbonSessionInfo.clone()
val currentThreadSessionInfoOrig = ThreadLocalSessionInfo.getCarbonSessionInfo
if (currentThreadSessionInfoOrig != null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index 6af0e98..fb0f9fe 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -24,7 +24,7 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.{CarbonEnv, Row, SparkSession, SQLContext}
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.execution.command.{AlterTableModel, CarbonMergerMapping, CompactionModel, DataCommand}
+import org.apache.spark.sql.execution.command.{AlterTableModel, AtomicRunnableCommand, CarbonMergerMapping, CompactionModel, DataCommand}
import org.apache.spark.sql.hive.{CarbonRelation, CarbonSessionCatalog}
import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.util.CarbonException
@@ -37,9 +37,10 @@ import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.events.{AlterTableCompactionPostEvent, AlterTableCompactionPreEvent, AlterTableCompactionPreStatusUpdateEvent, OperationContext, OperationListenerBus}
+import org.apache.carbondata.processing.loading.events.LoadEvents.LoadMetadataEvent
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CompactionType}
import org.apache.carbondata.spark.exception.ConcurrentOperationException
@@ -53,34 +54,42 @@ import org.apache.carbondata.streaming.segment.StreamSegment
*/
case class CarbonAlterTableCompactionCommand(
alterTableModel: AlterTableModel,
- tableInfoOp: Option[TableInfo] = None)
- extends DataCommand {
+ tableInfoOp: Option[TableInfo] = None,
+ val operationContext: OperationContext = new OperationContext ) extends AtomicRunnableCommand {
- override def processData(sparkSession: SparkSession): Seq[Row] = {
- val LOGGER: LogService =
- LogServiceFactory.getLogService(this.getClass.getName)
- val tableName = alterTableModel.tableName.toLowerCase
- val databaseName = alterTableModel.dbName.getOrElse(sparkSession.catalog.currentDatabase)
+ var table: CarbonTable = _
- val table = if (tableInfoOp.isDefined) {
- val tableInfo = tableInfoOp.get
- // To DO: CarbonEnv.updateStorePath
- CarbonTable.buildFromTableInfo(tableInfo)
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
+ val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ val tableName = alterTableModel.tableName.toLowerCase
+ val dbName = alterTableModel.dbName.getOrElse(sparkSession.catalog.currentDatabase)
+ table = if (tableInfoOp.isDefined) {
+ CarbonTable.buildFromTableInfo(tableInfoOp.get)
} else {
- val relation =
- CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Option(databaseName), tableName)(sparkSession)
- .asInstanceOf[CarbonRelation]
+ val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
if (relation == null) {
- throw new NoSuchTableException(databaseName, tableName)
+ throw new NoSuchTableException(dbName, tableName)
}
if (null == relation.carbonTable) {
- LOGGER.error(s"alter table failed. table not found: $databaseName.$tableName")
- throw new NoSuchTableException(databaseName, tableName)
+ LOGGER.error(s"Data loading failed. table not found: $dbName.$tableName")
+ throw new NoSuchTableException(dbName, tableName)
}
relation.carbonTable
}
+ if (CarbonUtil.hasAggregationDataMap(table) ||
+ (table.isChildDataMap && null == operationContext.getProperty(table.getTableName))) {
+ val loadMetadataEvent = new LoadMetadataEvent(table, true)
+ OperationListenerBus.getInstance().fireEvent(loadMetadataEvent, operationContext)
+ }
+ Seq.empty
+ }
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ val LOGGER: LogService =
+ LogServiceFactory.getLogService(this.getClass.getName)
+ val tableName = alterTableModel.tableName.toLowerCase
+ val databaseName = alterTableModel.dbName.getOrElse(sparkSession.catalog.currentDatabase)
val isLoadInProgress = SegmentStatusManager.checkIfAnyLoadInProgressForTable(table)
if (isLoadInProgress) {
val message = "Cannot run data loading and compaction on same table concurrently. " +
@@ -88,7 +97,6 @@ case class CarbonAlterTableCompactionCommand(
LOGGER.error(message)
throw new ConcurrentOperationException(message)
}
-
val carbonLoadModel = new CarbonLoadModel()
carbonLoadModel.setTableName(table.getTableName)
val dataLoadSchema = new CarbonDataLoadSchema(table)
@@ -103,7 +111,6 @@ case class CarbonAlterTableCompactionCommand(
System.getProperty("java.io.tmpdir"))
storeLocation = storeLocation + "/carbonstore/" + System.nanoTime()
// trigger event for compaction
- val operationContext = new OperationContext
val alterTableCompactionPreEvent: AlterTableCompactionPreEvent =
AlterTableCompactionPreEvent(sparkSession, table, null, null)
OperationListenerBus.getInstance.fireEvent(alterTableCompactionPreEvent, operationContext)
@@ -242,8 +249,6 @@ case class CarbonAlterTableCompactionCommand(
compactionModel.currentPartitions,
null)
- // trigger event for merge index
- val operationContext = new OperationContext
// trigger event for compaction
val alterTableCompactionPreStatusUpdateEvent: AlterTableCompactionPreStatusUpdateEvent =
AlterTableCompactionPreStatusUpdateEvent(sqlContext.sparkSession,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
index 810b10f..626cdba 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.command.management
import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Dataset, Row, SparkSession}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.command.DataCommand
+import org.apache.spark.sql.execution.command.{AtomicRunnableCommand, DataCommand}
import org.apache.carbondata.spark.util.CarbonSparkUtil
@@ -28,12 +28,14 @@ case class CarbonInsertIntoCommand(
child: LogicalPlan,
overwrite: Boolean,
partition: Map[String, Option[String]])
- extends DataCommand {
+ extends AtomicRunnableCommand {
- override def processData(sparkSession: SparkSession): Seq[Row] = {
+ var loadCommand: CarbonLoadDataCommand = _
+
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val df = Dataset.ofRows(sparkSession, child)
val header = relation.tableSchema.get.fields.map(_.name).mkString(",")
- val load = CarbonLoadDataCommand(
+ loadCommand = CarbonLoadDataCommand(
databaseNameOp = Some(relation.carbonRelation.databaseName),
tableName = relation.carbonRelation.tableName,
factPathFromUser = null,
@@ -45,10 +47,14 @@ case class CarbonInsertIntoCommand(
updateModel = None,
tableInfoOp = None,
internalOptions = Map.empty,
- partition = partition).run(sparkSession)
- // updating relation metadata. This is in case of auto detect high cardinality
- relation.carbonRelation.metaData =
- CarbonSparkUtil.createSparkMeta(relation.carbonRelation.carbonTable)
- load
+ partition = partition)
+ loadCommand.processMetadata(sparkSession)
+ }
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ if (null != loadCommand) {
+ loadCommand.processData(sparkSession)
+ } else {
+ Seq.empty
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 6b43152..7afbd92 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -39,15 +39,14 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference,
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan, Project}
import org.apache.spark.sql.execution.LogicalRDD
import org.apache.spark.sql.execution.SQLExecution.EXECUTION_ID_KEY
-import org.apache.spark.sql.execution.command.{DataCommand, DataLoadTableFileMapping, UpdateTableModel}
+import org.apache.spark.sql.execution.command.{AtomicRunnableCommand, DataLoadTableFileMapping, UpdateTableModel}
import org.apache.spark.sql.execution.datasources.{CarbonFileFormat, CatalogFileIndex, HadoopFsRelation, LogicalRelation}
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.optimizer.CarbonFilters
-import org.apache.spark.sql.types.{StringType, StructField, StructType, TimestampType}
+import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
-import org.apache.carbondata.common.constants.LoggerAction
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
import org.apache.carbondata.core.datamap.DataMapStoreManager
@@ -57,26 +56,24 @@ import org.apache.carbondata.core.dictionary.service.NonSecureDictionaryServiceP
import org.apache.carbondata.core.metadata.PartitionMapFileStore
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.{CarbonStorePath}
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.events.exception.PreEventException
-import org.apache.carbondata.format
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil
import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.csvinput.{CSVInputFormat, StringArrayWritable}
-import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
-import org.apache.carbondata.processing.loading.exception.{BadRecordFoundException, NoRetryException}
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
+import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadMetadataEvent, LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
+import org.apache.carbondata.processing.loading.exception.{NoRetryException}
+import org.apache.carbondata.processing.loading.model.{CarbonLoadModel}
import org.apache.carbondata.processing.util.CarbonLoaderUtil
import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD, DictionaryLoadModel}
+import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, CarbonDropPartitionCommitRDD, CarbonDropPartitionRDD}
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, DataLoadingUtil, GlobalDictionaryUtil}
case class CarbonLoadDataCommand(
@@ -87,12 +84,41 @@ case class CarbonLoadDataCommand(
options: scala.collection.immutable.Map[String, String],
isOverwriteTable: Boolean,
var inputSqlString: String = null,
- dataFrame: Option[DataFrame] = None,
+ var dataFrame: Option[DataFrame] = None,
updateModel: Option[UpdateTableModel] = None,
var tableInfoOp: Option[TableInfo] = None,
- internalOptions: Map[String, String] = Map.empty,
- partition: Map[String, Option[String]] = Map.empty) extends DataCommand {
+ var internalOptions: Map[String, String] = Map.empty,
+ partition: Map[String, Option[String]] = Map.empty,
+ logicalPlan: Option[LogicalPlan] = None,
+ var operationContext: OperationContext = new OperationContext) extends AtomicRunnableCommand {
+ var table: CarbonTable = _
+
+ override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
+ val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ val dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
+ table = if (tableInfoOp.isDefined) {
+ CarbonTable.buildFromTableInfo(tableInfoOp.get)
+ } else {
+ val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
+ if (relation == null) {
+ throw new NoSuchTableException(dbName, tableName)
+ }
+ if (null == relation.carbonTable) {
+ LOGGER.error(s"Data loading failed. table not found: $dbName.$tableName")
+ LOGGER.audit(s"Data loading failed. table not found: $dbName.$tableName")
+ throw new NoSuchTableException(dbName, tableName)
+ }
+ relation.carbonTable
+ }
+ operationContext.setProperty("isOverwrite", isOverwriteTable)
+ if(CarbonUtil.hasAggregationDataMap(table)) {
+ val loadMetadataEvent = new LoadMetadataEvent(table, false)
+ OperationListenerBus.getInstance().fireEvent(loadMetadataEvent, operationContext)
+ }
+ Seq.empty
+ }
override def processData(sparkSession: SparkSession): Seq[Row] = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
val carbonProperty: CarbonProperties = CarbonProperties.getInstance()
@@ -121,22 +147,6 @@ case class CarbonLoadDataCommand(
val hadoopConf = sparkSession.sessionState.newHadoopConf()
val carbonLoadModel = new CarbonLoadModel()
try {
- val table = if (tableInfoOp.isDefined) {
- CarbonTable.buildFromTableInfo(tableInfoOp.get)
- } else {
- val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
- if (relation == null) {
- throw new NoSuchTableException(dbName, tableName)
- }
- if (null == relation.carbonTable) {
- LOGGER.error(s"Data loading failed. table not found: $dbName.$tableName")
- LOGGER.audit(s"Data loading failed. table not found: $dbName.$tableName")
- throw new NoSuchTableException(dbName, tableName)
- }
- relation.carbonTable
- }
-
val tableProperties = table.getTableInfo.getFactTable.getTableProperties
val optionsFinal = DataLoadingUtil.getDataLoadingOptions(carbonProperty, options)
optionsFinal.put("sort_scope", tableProperties.asScala.getOrElse("sort_scope",
@@ -167,7 +177,6 @@ case class CarbonLoadDataCommand(
TableProcessingOperations.deletePartialLoadDataIfExist(table, false)
var isUpdateTableStatusRequired = false
try {
- val operationContext = new OperationContext
val loadTablePreExecutionEvent: LoadTablePreExecutionEvent =
new LoadTablePreExecutionEvent(
table.getCarbonTableIdentifier,
@@ -181,7 +190,6 @@ case class CarbonLoadDataCommand(
OperationListenerBus.getInstance.fireEvent(loadTablePreExecutionEvent, operationContext)
// First system has to partition the data first and then call the load data
LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
- GlobalDictionaryUtil.updateTableMetadataFunc = updateTableMetadata
// Clean up the old invalid segment data before creating a new entry for new load.
DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false, table)
// add the start entry for the new load in the table status file
@@ -525,6 +533,7 @@ case class CarbonLoadDataCommand(
CarbonSession.threadSet(
CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
isEmptyBadRecord)
+ CarbonSession.threadSet("partition.operationcontext", operationContext)
try {
val query: LogicalPlan = if (dataFrame.isDefined) {
val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
@@ -649,6 +658,7 @@ case class CarbonLoadDataCommand(
CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_SERIALIZATION_NULL_FORMAT)
CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
CarbonSession.threadUnset(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
+ CarbonSession.threadUnset("partition.operationcontext")
}
try {
// Trigger auto compaction
@@ -718,7 +728,6 @@ case class CarbonLoadDataCommand(
val dataSchema =
StructType(metastoreSchema
.filterNot(field => partitionSchema.contains(field.name)))
- val operationContextStr = ObjectSerializationUtil.convertObjectToString(operationContext)
val options = new mutable.HashMap[String, String]()
options ++= catalogTable.storage.properties
options += (("overwrite", overWriteLocal.toString))
@@ -731,7 +740,6 @@ case class CarbonLoadDataCommand(
partition.map{case (col, value) => (col.toLowerCase, value.isDefined)}.asJava))
options += (("staticpartition", staticPartitionStr))
}
- options += (("operationcontext", operationContextStr))
options ++= this.options
if (updateModel.isDefined) {
options += (("updatetimestamp", updateModel.get.updatedTimeStamp.toString))
@@ -869,41 +877,4 @@ case class CarbonLoadDataCommand(
val dataFrameWithTupleId = dataFrame.get.select(fieldWithTupleId: _*)
(dataFrameWithTupleId)
}
-
- private def updateTableMetadata(
- carbonLoadModel: CarbonLoadModel,
- sqlContext: SQLContext,
- model: DictionaryLoadModel,
- noDictDimension: Array[CarbonDimension]): Unit = {
- val sparkSession = sqlContext.sparkSession
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(model.table)
-
- val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- // read TableInfo
- val tableInfo: format.TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
-
- // modify TableInfo
- val columns = tableInfo.getFact_table.getTable_columns
- for (i <- 0 until columns.size) {
- if (noDictDimension.exists(x => columns.get(i).getColumn_id.equals(x.getColumnId))) {
- columns.get(i).encoders.remove(org.apache.carbondata.format.Encoding.DICTIONARY)
- }
- }
- val entry = tableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history.get(0)
- entry.setTime_stamp(System.currentTimeMillis())
-
- // write TableInfo
- metastore.updateTableSchemaForAlter(carbonTablePath.getCarbonTableIdentifier,
- carbonTablePath.getCarbonTableIdentifier,
- tableInfo, entry, carbonTablePath.getPath)(sparkSession)
-
- // update the schema modified time
- metastore.updateAndTouchSchemasUpdatedTime()
-
- val identifier = model.table.getCarbonTableIdentifier
- // update CarbonDataLoadSchema
- val carbonTable = metastore.lookupRelation(Option(identifier.getDatabaseName),
- identifier.getTableName)(sqlContext.sparkSession).asInstanceOf[CarbonRelation].carbonTable
- carbonLoadModel.setCarbonDataLoadSchema(new CarbonDataLoadSchema(carbonTable))
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index 56f298a..c5340c2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -24,13 +24,13 @@ import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.command.datamap.CarbonDropDataMapCommand
+import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
import org.apache.spark.sql.execution.command.table.CarbonCreateTableCommand
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable}
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
/**
@@ -51,6 +51,7 @@ case class CreatePreAggregateTableCommand(
extends AtomicRunnableCommand {
var parentTable: CarbonTable = _
+ var loadCommand: CarbonLoadDataCommand = _
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val updatedQuery = new CarbonSpark2SqlParser().addPreAggFunction(queryString)
@@ -64,8 +65,6 @@ case class CreatePreAggregateTableCommand(
parentTable = PreAggregateUtil.getParentCarbonTable(df.logicalPlan)
assert(parentTable.getTableName.equalsIgnoreCase(parentTableIdentifier.table),
"Parent table name is different in select and create")
-
-
var neworder = Seq[String]()
val parentOrder = parentTable.getSortColumns(parentTable.getTableName).asScala
parentOrder.foreach(parentcol =>
@@ -80,7 +79,9 @@ case class CreatePreAggregateTableCommand(
.LOAD_SORT_SCOPE_DEFAULT))
tableProperties
.put(CarbonCommonConstants.TABLE_BLOCKSIZE, parentTable.getBlockSizeInMB.toString)
-
+ val tableIdentifier =
+ TableIdentifier(parentTableIdentifier.table + "_" + dataMapName,
+ parentTableIdentifier.database)
// prepare table model of the collected tokens
val tableModel: TableModel = new CarbonSpark2SqlParser().prepareTableModel(
ifNotExistPresent = false,
@@ -137,7 +138,30 @@ case class CreatePreAggregateTableCommand(
// to be used in further create process.
parentTable = CarbonEnv.getCarbonTable(parentTableIdentifier.database,
parentTableIdentifier.table)(sparkSession)
-
+ val updatedLoadQuery = if (timeSeriesFunction.isDefined) {
+ val dataMap = parentTable.getTableInfo.getDataMapSchemaList.asScala
+ .filter(p => p.getDataMapName
+ .equalsIgnoreCase(dataMapName)).head
+ .asInstanceOf[AggregationDataMapSchema]
+ PreAggregateUtil.createTimeSeriesSelectQueryFromMain(dataMap.getChildSchema,
+ parentTable.getTableName,
+ parentTable.getDatabaseName)
+ }
+ else {
+ queryString
+ }
+ val dataFrame = sparkSession.sql(new CarbonSpark2SqlParser().addPreAggLoadFunction(
+ updatedLoadQuery)).drop("preAggLoad")
+ val dataMap = parentTable.getTableInfo.getDataMapSchemaList.asScala
+ .filter(dataMap => dataMap.getDataMapName.equalsIgnoreCase(dataMapName)).head
+ .asInstanceOf[AggregationDataMapSchema]
+ loadCommand = PreAggregateUtil.createLoadCommandForChild(
+ dataMap.getChildSchema.getListOfColumns,
+ tableIdentifier,
+ dataFrame,
+ false,
+ sparkSession = sparkSession)
+ loadCommand.processMetadata(sparkSession)
Seq.empty
}
@@ -159,35 +183,19 @@ case class CreatePreAggregateTableCommand(
val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetaDataFilepath)
.nonEmpty
if (loadAvailable) {
- val updatedQuery = if (timeSeriesFunction.isDefined) {
- val dataMap = parentTable.getTableInfo.getDataMapSchemaList.asScala
- .filter(p => p.getDataMapName
- .equalsIgnoreCase(dataMapName)).head
- .asInstanceOf[AggregationDataMapSchema]
- PreAggregateUtil.createTimeSeriesSelectQueryFromMain(dataMap.getChildSchema,
- parentTable.getTableName,
- parentTable.getDatabaseName)
- } else {
- queryString
- }
// Passing segmentToLoad as * because we want to load all the segments into the
// pre-aggregate table even if the user has set some segments on the parent table.
+ loadCommand.dataFrame = Some(PreAggregateUtil
+ .getDataFrame(sparkSession, loadCommand.logicalPlan.get))
PreAggregateUtil.startDataLoadForDataMap(
- parentTable,
- tableIdentifier,
- updatedQuery,
- segmentToLoad = "*",
- validateSegments = true,
- isOverwrite = false,
- sparkSession = sparkSession)
+ parentTable,
+ segmentToLoad = "*",
+ validateSegments = true,
+ sparkSession,
+ loadCommand)
}
Seq.empty
}
-
- // Create the aggregation table name with parent table name prefix
- private lazy val tableIdentifier =
- TableIdentifier(parentTableIdentifier.table + "_" + dataMapName, parentTableIdentifier.database)
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
index fce32ab..7b273ba 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
@@ -20,18 +20,153 @@ package org.apache.spark.sql.execution.command.preaaggregate
import scala.collection.JavaConverters._
import scala.collection.mutable
-import org.apache.spark.sql.CarbonEnv
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.{SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.AlterTableModel
-import org.apache.spark.sql.execution.command.management.CarbonAlterTableCompactionCommand
+import org.apache.spark.sql.execution.command.management.{CarbonAlterTableCompactionCommand, CarbonLoadDataCommand}
+import org.apache.spark.sql.parser.CarbonSpark2SqlParser
-import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.events._
-import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePreExecutionEvent, LoadTablePreStatusUpdateEvent}
+import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadMetadataEvent, LoadTablePreExecutionEvent, LoadTablePreStatusUpdateEvent}
+/**
+ * below class will be used to create load command for compaction
+ * for all the pre agregate child data map
+ */
+object CompactionProcessMetaListener extends OperationEventListener {
+ /**
+ * Called on a specified event occurrence
+ *
+ * @param event
+ * @param operationContext
+ */
+ override protected def onEvent(event: Event,
+ operationContext: OperationContext): Unit = {
+ val sparkSession = SparkSession.getActiveSession.get
+ val tableEvent = event.asInstanceOf[LoadMetadataEvent]
+ val table = tableEvent.getCarbonTable
+ if (!table.isChildDataMap && CarbonUtil.hasAggregationDataMap(table)) {
+ val aggregationDataMapList = table.getTableInfo.getDataMapSchemaList.asScala
+ .filter(_.isInstanceOf[AggregationDataMapSchema])
+ .asInstanceOf[mutable.ArrayBuffer[AggregationDataMapSchema]]
+ for (dataMapSchema: AggregationDataMapSchema <- aggregationDataMapList) {
+ val childTableName = dataMapSchema.getRelationIdentifier.getTableName
+ val childDatabaseName = dataMapSchema.getRelationIdentifier.getDatabaseName
+ // Creating a new query string to insert data into pre-aggregate table from that same table.
+ // For example: To compact preaggtable1 we can fire a query like insert into preaggtable1
+ // select * from preaggtable1
+ // The following code will generate the select query with a load UDF that will be used to
+ // apply DataLoadingRules
+ val childDataFrame = sparkSession.sql(new CarbonSpark2SqlParser()
+ // adding the aggregation load UDF
+ .addPreAggLoadFunction(
+ // creating the select query on the bases on table schema
+ PreAggregateUtil.createChildSelectQuery(
+ dataMapSchema.getChildSchema, table.getDatabaseName))).drop("preAggLoad")
+ val loadCommand = PreAggregateUtil.createLoadCommandForChild(
+ dataMapSchema.getChildSchema.getListOfColumns,
+ TableIdentifier(childTableName, Some(childDatabaseName)),
+ childDataFrame,
+ false,
+ sparkSession)
+ loadCommand.processMetadata(sparkSession)
+ operationContext
+ .setProperty(dataMapSchema.getChildSchema.getTableName + "_Compaction", loadCommand)
+ }
+ } else if (table.isChildDataMap) {
+ val childTableName = table.getTableName
+ val childDatabaseName = table.getDatabaseName
+ // Creating a new query string to insert data into pre-aggregate table from that same table.
+ // For example: To compact preaggtable1 we can fire a query like insert into preaggtable1
+ // select * from preaggtable1
+ // The following code will generate the select query with a load UDF that will be used to
+ // apply DataLoadingRules
+ val childDataFrame = sparkSession.sql(new CarbonSpark2SqlParser()
+ // adding the aggregation load UDF
+ .addPreAggLoadFunction(
+ // creating the select query on the bases on table schema
+ PreAggregateUtil.createChildSelectQuery(
+ table.getTableInfo.getFactTable, table.getDatabaseName))).drop("preAggLoad")
+ val loadCommand = PreAggregateUtil.createLoadCommandForChild(
+ table.getTableInfo.getFactTable.getListOfColumns,
+ TableIdentifier(childTableName, Some(childDatabaseName)),
+ childDataFrame,
+ false,
+ sparkSession)
+ loadCommand.processMetadata(sparkSession)
+ operationContext.setProperty(table.getTableName + "_Compaction", loadCommand)
+ }
+ }
+}
+
+/**
+ * Below class to is to create LoadCommand for loading the
+ * the data of pre aggregate data map
+ */
+object LoadProcessMetaListener extends OperationEventListener {
+ /**
+ * Called on a specified event occurrence
+ *
+ * @param event
+ * @param operationContext
+ */
+ override protected def onEvent(event: Event,
+ operationContext: OperationContext): Unit = {
+ val sparkSession = SparkSession.getActiveSession.get
+ val tableEvent = event.asInstanceOf[LoadMetadataEvent]
+ if (!tableEvent.isCompaction) {
+ val table = tableEvent.getCarbonTable
+ if (CarbonUtil.hasAggregationDataMap(table)) {
+ // getting all the aggergate datamap schema
+ val aggregationDataMapList = table.getTableInfo.getDataMapSchemaList.asScala
+ .filter(_.isInstanceOf[AggregationDataMapSchema])
+ .asInstanceOf[mutable.ArrayBuffer[AggregationDataMapSchema]]
+ // sorting the datamap for timeseries rollup
+ val sortedList = aggregationDataMapList.sortBy(_.getOrdinal)
+ val parentTableName = table.getTableName
+ val databaseName = table.getDatabaseName
+ val list = scala.collection.mutable.ListBuffer.empty[AggregationDataMapSchema]
+ for (dataMapSchema: AggregationDataMapSchema <- sortedList) {
+ val childTableName = dataMapSchema.getRelationIdentifier.getTableName
+ val childDatabaseName = dataMapSchema.getRelationIdentifier.getDatabaseName
+ val childSelectQuery = if (!dataMapSchema.isTimeseriesDataMap) {
+ PreAggregateUtil.getChildQuery(dataMapSchema)
+ } else {
+ // for timeseries rollup policy
+ val tableSelectedForRollup = PreAggregateUtil.getRollupDataMapNameForTimeSeries(list,
+ dataMapSchema)
+ list += dataMapSchema
+ // if non of the rollup data map is selected hit the maintable and prepare query
+ if (tableSelectedForRollup.isEmpty) {
+ PreAggregateUtil.createTimeSeriesSelectQueryFromMain(dataMapSchema.getChildSchema,
+ parentTableName,
+ databaseName)
+ } else {
+ // otherwise hit the select rollup datamap schema
+ PreAggregateUtil.createTimeseriesSelectQueryForRollup(dataMapSchema.getChildSchema,
+ tableSelectedForRollup.get,
+ databaseName)
+ }
+ }
+ val childDataFrame = sparkSession.sql(new CarbonSpark2SqlParser().addPreAggLoadFunction(
+ childSelectQuery)).drop("preAggLoad")
+ val isOverwrite =
+ operationContext.getProperty("isOverwrite").asInstanceOf[Boolean]
+ val loadCommand = PreAggregateUtil.createLoadCommandForChild(
+ dataMapSchema.getChildSchema.getListOfColumns,
+ TableIdentifier(childTableName, Some(childDatabaseName)),
+ childDataFrame,
+ isOverwrite,
+ sparkSession)
+ loadCommand.processMetadata(sparkSession)
+ operationContext.setProperty(dataMapSchema.getChildSchema.getTableName, loadCommand)
+ }
+ }
+ }
+ }
+}
object LoadPostAggregateListener extends OperationEventListener {
/**
* Called on a specified event occurrence
@@ -42,8 +177,7 @@ object LoadPostAggregateListener extends OperationEventListener {
val loadEvent = event.asInstanceOf[LoadTablePreStatusUpdateEvent]
val sparkSession = SparkSession.getActiveSession.get
val carbonLoadModel = loadEvent.getCarbonLoadModel
- val table = CarbonEnv.getCarbonTable(Option(carbonLoadModel.getDatabaseName),
- carbonLoadModel.getTableName)(sparkSession)
+ val table = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
if (CarbonUtil.hasAggregationDataMap(table)) {
// getting all the aggergate datamap schema
val aggregationDataMapList = table.getTableInfo.getDataMapSchemaList.asScala
@@ -51,41 +185,28 @@ object LoadPostAggregateListener extends OperationEventListener {
.asInstanceOf[mutable.ArrayBuffer[AggregationDataMapSchema]]
// sorting the datamap for timeseries rollup
val sortedList = aggregationDataMapList.sortBy(_.getOrdinal)
- val parentTableName = table.getTableName
- val databasename = table.getDatabaseName
- val list = scala.collection.mutable.ListBuffer.empty[AggregationDataMapSchema]
for (dataMapSchema: AggregationDataMapSchema <- sortedList) {
- val childTableName = dataMapSchema.getRelationIdentifier.getTableName
- val childDatabaseName = dataMapSchema.getRelationIdentifier.getDatabaseName
- val childSelectQuery = if (!dataMapSchema.isTimeseriesDataMap) {
- PreAggregateUtil.getChildQuery(dataMapSchema)
- } else {
- // for timeseries rollup policy
- val tableSelectedForRollup = PreAggregateUtil.getRollupDataMapNameForTimeSeries(list,
- dataMapSchema)
- list += dataMapSchema
- // if non of the rollup data map is selected hit the maintable and prepare query
- if (tableSelectedForRollup.isEmpty) {
- PreAggregateUtil.createTimeSeriesSelectQueryFromMain(dataMapSchema.getChildSchema,
- parentTableName,
- databasename)
- } else {
- // otherwise hit the select rollup datamap schema
- PreAggregateUtil.createTimeseriesSelectQueryForRollup(dataMapSchema.getChildSchema,
- tableSelectedForRollup.get,
- databasename)
- }
- }
+ val childLoadCommand = operationContext
+ .getProperty(dataMapSchema.getChildSchema.getTableName)
+ .asInstanceOf[CarbonLoadDataCommand]
+ childLoadCommand.dataFrame = Some(PreAggregateUtil
+ .getDataFrame(sparkSession, childLoadCommand.logicalPlan.get))
+ val childOperationContext = new OperationContext
+ childOperationContext
+ .setProperty(dataMapSchema.getChildSchema.getTableName,
+ operationContext.getProperty(dataMapSchema.getChildSchema.getTableName))
val isOverwrite =
operationContext.getProperty("isOverwrite").asInstanceOf[Boolean]
+ childOperationContext.setProperty("isOverwrite", isOverwrite)
+ childOperationContext.setProperty(dataMapSchema.getChildSchema.getTableName + "_Compaction",
+ operationContext.getProperty(dataMapSchema.getChildSchema.getTableName + "_Compaction"))
+ childLoadCommand.operationContext = childOperationContext
PreAggregateUtil.startDataLoadForDataMap(
table,
- TableIdentifier(childTableName, Some(childDatabaseName)),
- childSelectQuery,
carbonLoadModel.getSegmentId,
validateSegments = false,
- isOverwrite,
- sparkSession)
+ sparkSession,
+ childLoadCommand)
}
}
}
@@ -115,7 +236,8 @@ object AlterPreAggregateTableCompactionPostListener extends OperationEventListen
compactionType.toString,
Some(System.currentTimeMillis()),
"")
- CarbonAlterTableCompactionCommand(alterTableModel).run(sparkSession)
+ CarbonAlterTableCompactionCommand(alterTableModel, operationContext = operationContext)
+ .run(sparkSession)
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index cd19e3b..dac5d5e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.command.preaaggregate
import scala.collection.mutable.{ArrayBuffer, ListBuffer}
import scala.collection.JavaConverters._
-import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, CarbonSession, SparkSession}
+import org.apache.spark.sql._
import org.apache.spark.sql.CarbonExpressions.{CarbonSubqueryAlias => SubqueryAlias, MatchCastExpression}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedFunction, UnresolvedRelation}
@@ -31,13 +31,14 @@ import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
-import org.apache.spark.sql.types.{DataType, LongType}
+import org.apache.spark.sql.types.DataType
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable, DataMapSchema, TableSchema}
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.format.TableInfo
@@ -576,18 +577,15 @@ object PreAggregateUtil {
}
updatedPlan
}
-
- /**
+ /**
* This method will start load process on the data map
*/
def startDataLoadForDataMap(
parentCarbonTable: CarbonTable,
- dataMapIdentifier: TableIdentifier,
- queryString: String,
segmentToLoad: String,
validateSegments: Boolean,
- isOverwrite: Boolean,
- sparkSession: SparkSession): Unit = {
+ sparkSession: SparkSession,
+ loadCommand: CarbonLoadDataCommand): Unit = {
CarbonSession.threadSet(
CarbonCommonConstants.CARBON_INPUT_SEGMENTS +
parentCarbonTable.getDatabaseName + "." +
@@ -597,32 +595,9 @@ object PreAggregateUtil {
CarbonCommonConstants.VALIDATE_CARBON_INPUT_SEGMENTS +
parentCarbonTable.getDatabaseName + "." +
parentCarbonTable.getTableName, validateSegments.toString)
- val dataMapSchemas = parentCarbonTable.getTableInfo.getDataMapSchemaList.asScala
- val headers = dataMapSchemas.find(_.getChildSchema.getTableName.equalsIgnoreCase(
- dataMapIdentifier.table)) match {
- case Some(dataMapSchema) =>
- val columns = dataMapSchema.getChildSchema.getListOfColumns.asScala
- .filter{column =>
- !column.getColumnName.equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)}
- columns.sortBy(_.getSchemaOrdinal).map(
- _.getColumnName).mkString(",")
- case None =>
- throw new RuntimeException(
- s"${ dataMapIdentifier.table} datamap not found in DataMapSchema list: ${
- dataMapSchemas.map(_.getChildSchema.getTableName).mkString("[", ",", "]")}")
- }
- val dataFrame = sparkSession.sql(new CarbonSpark2SqlParser().addPreAggLoadFunction(
- queryString)).drop("preAggLoad")
+ CarbonSession.updateSessionInfoToCurrentThread(sparkSession)
try {
- CarbonLoadDataCommand(dataMapIdentifier.database,
- dataMapIdentifier.table,
- null,
- Nil,
- Map("fileheader" -> headers),
- isOverwriteTable = isOverwrite,
- dataFrame = Some(dataFrame),
- internalOptions = Map(CarbonCommonConstants.IS_INTERNAL_LOAD_CALL -> "true")).
- run(sparkSession)
+ loadCommand.processData(sparkSession)
} finally {
CarbonSession.threadUnset(
CarbonCommonConstants.CARBON_INPUT_SEGMENTS +
@@ -647,11 +622,12 @@ object PreAggregateUtil {
case _ => a.getAggFunction}}(${a.getColumnName})"
} else {
groupingExpressions += a.getColumnName
+ aggregateColumns+= a.getColumnName
}
}
- s"select ${ groupingExpressions.mkString(",") },${ aggregateColumns.mkString(",")
- } from $databaseName.${ tableSchema.getTableName } group by ${
- groupingExpressions.mkString(",") }"
+ s"select ${ aggregateColumns.mkString(",") } " +
+ s"from $databaseName.${ tableSchema.getTableName }" +
+ s" group by ${ groupingExpressions.mkString(",") }"
}
/**
@@ -900,4 +876,32 @@ object PreAggregateUtil {
aggDataMapSchema.getProperties.get("CHILD_SELECT QUERY").replace("&", "=")),
CarbonCommonConstants.DEFAULT_CHARSET)
}
+
+ /**
+ * This method will start load process on the data map
+ */
+ def createLoadCommandForChild(
+ columns: java.util.List[ColumnSchema],
+ dataMapIdentifier: TableIdentifier,
+ dataFrame: DataFrame,
+ isOverwrite: Boolean,
+ sparkSession: SparkSession): CarbonLoadDataCommand = {
+ val headers = columns.asScala.filter { column =>
+ !column.getColumnName.equalsIgnoreCase(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)
+ }.sortBy(_.getSchemaOrdinal).map(_.getColumnName).mkString(",")
+ val loadCommand = CarbonLoadDataCommand(dataMapIdentifier.database,
+ dataMapIdentifier.table,
+ null,
+ Nil,
+ Map("fileheader" -> headers),
+ isOverwriteTable = isOverwrite,
+ dataFrame = None,
+ internalOptions = Map(CarbonCommonConstants.IS_INTERNAL_LOAD_CALL -> "true"),
+ logicalPlan = Some(dataFrame.queryExecution.logical))
+ loadCommand
+ }
+
+ def getDataFrame(sparkSession: SparkSession, child: LogicalPlan): DataFrame = {
+ Dataset.ofRows(sparkSession, child)
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
index 26a8f6f..19c265d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableAsSelectCommand.scala
@@ -42,15 +42,12 @@ case class CarbonCreateTableAsSelectCommand(
ifNotExistsSet: Boolean = false,
tableLocation: Option[String] = None) extends AtomicRunnableCommand {
- /**
- * variable to be used for insert into command for checking whether the
- * table is created newly or already existed
- */
- var isTableCreated: Boolean = false
+ var loadCommand: CarbonInsertIntoCommand = _
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
val tableName = tableInfo.getFactTable.getTableName
+ var isTableCreated = false
var databaseOpt: Option[String] = None
if (tableInfo.getDatabaseName != null) {
databaseOpt = Some(tableInfo.getDatabaseName)
@@ -71,10 +68,7 @@ case class CarbonCreateTableAsSelectCommand(
CarbonCreateTableCommand(tableInfo, ifNotExistsSet, tableLocation).run(sparkSession)
isTableCreated = true
}
- Seq.empty
- }
- override def processData(sparkSession: SparkSession): Seq[Row] = {
if (isTableCreated) {
val tableName = tableInfo.getFactTable.getTableName
var databaseOpt: Option[String] = None
@@ -87,12 +81,23 @@ case class CarbonCreateTableAsSelectCommand(
.createCarbonDataSourceHadoopRelation(sparkSession,
TableIdentifier(tableName, Option(dbName)))
// execute command to load data into carbon table
- CarbonInsertIntoCommand(
+ loadCommand = CarbonInsertIntoCommand(
carbonDataSourceHadoopRelation,
query,
overwrite = false,
- partition = Map.empty).run(sparkSession)
- LOGGER.audit(s"CTAS operation completed successfully for $dbName.$tableName")
+ partition = Map.empty)
+ loadCommand.processMetadata(sparkSession)
+ }
+ Seq.empty
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ if (null != loadCommand) {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ loadCommand.processData(sparkSession)
+ val carbonTable = loadCommand.relation.carbonTable
+ LOGGER.audit(s"CTAS operation completed successfully for " +
+ s"${carbonTable.getDatabaseName}.${carbonTable.getTableName}")
}
Seq.empty
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index d74e461..99e5732 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -127,10 +127,6 @@ with Serializable {
if (segemntsTobeDeleted.isDefined) {
conf.set(CarbonTableOutputFormat.SEGMENTS_TO_BE_DELETED, segemntsTobeDeleted.get)
}
- val operationContextStr = options.get("operationcontext")
- if (operationContextStr.isDefined) {
- conf.set(CarbonTableOutputFormat.OPERATION_CONTEXT, operationContextStr.get)
- }
CarbonTableOutputFormat.setLoadModel(conf, model)
new OutputWriterFactory {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index f058e96..57be754 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -95,7 +95,7 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
ExecutedCommandExec(createDb) :: Nil
case drop@DropDatabaseCommand(dbName, ifExists, isCascade) =>
ExecutedCommandExec(CarbonDropDatabaseCommand(drop)) :: Nil
- case alterTable@CarbonAlterTableCompactionCommand(altertablemodel, _) =>
+ case alterTable@CarbonAlterTableCompactionCommand(altertablemodel, _, _) =>
val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
.tableExists(TableIdentifier(altertablemodel.tableName,
altertablemodel.dbName))(sparkSession)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
index 87be2d2..b8608f4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
@@ -34,7 +34,7 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
/**
* Represents logical plan for one carbon table
@@ -204,7 +204,6 @@ case class CarbonRelation(
def sizeInBytes: Long = {
val tableStatusNewLastUpdatedTime = SegmentStatusManager.getTableStatusLastModifiedTime(
carbonTable.getAbsoluteTableIdentifier)
-
if (tableStatusLastUpdateTime != tableStatusNewLastUpdatedTime) {
if (new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
.getValidAndInvalidSegments.getValidSegments.isEmpty) {
@@ -215,8 +214,19 @@ case class CarbonRelation(
carbonTable.getCarbonTableIdentifier).getPath
val fileType = FileFactory.getFileType(tablePath)
if (FileFactory.isFileExist(tablePath, fileType)) {
+ // get the valid segments
+ val segments = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
+ .getValidAndInvalidSegments.getValidSegments.asScala
+ var size = 0L
+ // for each segment calculate the size
+ segments.foreach {validSeg =>
+ size = size + FileFactory.getDirectorySize(
+ CarbonTablePath.getSegmentPath(tablePath, validSeg))
+ }
+ // update the new table status time
tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
- sizeInBytesLocalValue = FileFactory.getDirectorySize(tablePath)
+ // update the new size
+ sizeInBytesLocalValue = size
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3a6136df/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
index b00a67e..78964e7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/events/LoadEvents.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.processing.loading.events;
import java.util.Map;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.events.Event;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
@@ -129,6 +130,24 @@ public class LoadEvents {
}
/**
+ * Load Even class will be fired from the Load and compaction class
+ * to creating all the load commands for all preaggregate data map
+ */
+ public static class LoadMetadataEvent extends Event {
+ private CarbonTable carbonTable;
+ private boolean isCompaction;
+ public LoadMetadataEvent(CarbonTable carbonTable, boolean isCompaction) {
+ this.carbonTable = carbonTable;
+ this.isCompaction = isCompaction;
+ }
+ public boolean isCompaction() {
+ return isCompaction;
+ }
+ public CarbonTable getCarbonTable() {
+ return carbonTable;
+ }
+ }
+ /**
* Class for handling clean up in case of any failure and abort the operation.
*/
[03/50] [abbrv] carbondata git commit: [CARBONDATA-2051] Added like
query ends with and contains with filter push down suport to carbondata
Posted by ja...@apache.org.
[CARBONDATA-2051] Added like query ends with and contains with filter push down suport to carbondata
Problem
Current like filter with start with expression is only pushed down to carbondata. In case of ends with and contains like filter all the data is given back to spark and then spark applies the filter on it.
This behavior is fine for the queries which has lesser number of queried columns. But as the number of columns and data increases there is performance impact because the data being sent to spark becomes more thereby increasing the IO.
If like filter is push down then first filter column is read and blocks are pruned. In this cases the data returned to the spark is after applying the filter and only blocklets matching the data are fully read. This reduces IO and increases the query performance.
Solution
Modify code to push down like query with ends and contains with filter
This closes #1830
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/543a903c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/543a903c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/543a903c
Branch: refs/heads/carbonstore
Commit: 543a903c6993f553ada309ec3ae23e0385013422
Parents: 5ea538f
Author: manishgupta88 <to...@gmail.com>
Authored: Thu Jan 18 14:23:17 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Jan 18 22:55:08 2018 +0530
----------------------------------------------------------------------
.../filterexpr/AllDataTypesTestCaseFilter.scala | 12 ++++++++++++
.../apache/spark/sql/CarbonBoundReference.scala | 8 ++++++++
.../strategy/CarbonLateDecodeStrategy.scala | 4 ++++
.../spark/sql/optimizer/CarbonFilters.scala | 18 ++++++++++++++++++
4 files changed, 42 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/543a903c/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
index e90ed3c..15ac1f4 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
@@ -52,6 +52,18 @@ class AllDataTypesTestCaseFilter extends QueryTest with BeforeAndAfterAll {
sql("select empno,empname from alldatatypestableFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"),
sql("select empno,empname from alldatatypestableFilter_hive where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"))
}
+
+ test("verify like query ends with filter push down") {
+ val df = sql("select * from alldatatypestableFilter where empname like '%nandh'").queryExecution
+ .sparkPlan
+ assert(df.metadata.get("PushedFilters").get.contains("CarbonEndsWith"))
+ }
+
+ test("verify like query contains with filter push down") {
+ val df = sql("select * from alldatatypestableFilter where empname like '%nand%'").queryExecution
+ .sparkPlan
+ assert(df.metadata.get("PushedFilters").get.contains("CarbonContainsWith"))
+ }
override def afterAll {
sql("drop table alldatatypestableFilter")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/543a903c/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
index 21e56b0..a043342 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
@@ -48,3 +48,11 @@ case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nu
override def newInstance(): NamedExpression = throw new UnsupportedOperationException
}
+
+case class CarbonEndsWith(expr: Expression) extends Filter {
+ override def references: Array[String] = null
+}
+
+case class CarbonContainsWith(expr: Expression) extends Filter {
+ override def references: Array[String] = null
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/543a903c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index 6ef3d47..4b1d11b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -612,6 +612,10 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
CastExpressionOptimization.checkIfCastCanBeRemove(c)
case StartsWith(a: Attribute, Literal(v, t)) =>
Some(sources.StringStartsWith(a.name, v.toString))
+ case c@EndsWith(a: Attribute, Literal(v, t)) =>
+ Some(CarbonEndsWith(c))
+ case c@Contains(a: Attribute, Literal(v, t)) =>
+ Some(CarbonContainsWith(c))
case others => None
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/543a903c/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index 341b513..4d91375 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -23,6 +23,8 @@ import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.execution.CastExpressionOptimization
import org.apache.spark.sql.types._
+import org.apache.spark.sql.CarbonContainsWith
+import org.apache.spark.sql.CarbonEndsWith
import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.hive.CarbonSessionCatalog
@@ -119,6 +121,18 @@ object CarbonFilters {
val r = new LessThanExpression(
getCarbonExpression(name), getCarbonLiteralExpression(name, maxValueLimit))
Some(new AndExpression(l, r))
+ case CarbonEndsWith(expr: Expression) =>
+ Some(new SparkUnknownExpression(expr.transform {
+ case AttributeReference(name, dataType, _, _) =>
+ CarbonBoundReference(new CarbonColumnExpression(name.toString,
+ CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable)
+ }))
+ case CarbonContainsWith(expr: Expression) =>
+ Some(new SparkUnknownExpression(expr.transform {
+ case AttributeReference(name, dataType, _, _) =>
+ CarbonBoundReference(new CarbonColumnExpression(name.toString,
+ CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable)
+ }))
case CastExpr(expr: Expression) =>
Some(transformExpression(expr))
case _ => None
@@ -249,6 +263,10 @@ object CarbonFilters {
CastExpressionOptimization.checkIfCastCanBeRemove(c)
case StartsWith(a: Attribute, Literal(v, t)) =>
Some(sources.StringStartsWith(a.name, v.toString))
+ case c@EndsWith(a: Attribute, Literal(v, t)) =>
+ Some(CarbonEndsWith(c))
+ case c@Contains(a: Attribute, Literal(v, t)) =>
+ Some(CarbonContainsWith(c))
case c@Cast(a: Attribute, _) =>
Some(CastExpr(c))
case others =>
[18/50] [abbrv] carbondata git commit: [HOTFIX] process JVM error for
the streaming writer
Posted by ja...@apache.org.
[HOTFIX] process JVM error for the streaming writer
process JVM error for the streaming writer by using Throwable instead of Exception
This closes #1849
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d9bb6479
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d9bb6479
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d9bb6479
Branch: refs/heads/carbonstore
Commit: d9bb647978712d39d72273b816c2d3a84307854f
Parents: 3312ee7
Author: QiangCai <qi...@qq.com>
Authored: Tue Jan 23 21:12:32 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Jan 25 15:48:50 2018 +0800
----------------------------------------------------------------------
.../carbondata/hadoop/streaming/CarbonStreamRecordWriter.java | 3 ++-
.../org/apache/carbondata/streaming/segment/StreamSegment.java | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9bb6479/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
index 212baa7..364a6a6 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
@@ -134,7 +134,6 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
}
private void initializeAtFirstRow() throws IOException, InterruptedException {
- isFirstRow = false;
// initialize metadata
isNoDictionaryDimensionColumn =
@@ -173,6 +172,8 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
outputStream = FileFactory.getDataOutputStream(filePath, fileType);
writeFileHeader();
}
+
+ isFirstRow = false;
}
@Override public void write(Void key, Object value) throws IOException, InterruptedException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9bb6479/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
index 74e1461..4dbdfb0 100644
--- a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
+++ b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
@@ -244,7 +244,7 @@ public class StreamSegment {
writer.write(null, inputIterators.next());
}
inputIterators.close();
- } catch (Exception ex) {
+ } catch (Throwable ex) {
if (writer != null) {
LOGGER.error(ex, "Failed to append batch data to stream segment: " +
writer.getSegmentDir());
[47/50] [abbrv] carbondata git commit: [CARBONDATA-1992] Remove
partitionId in CarbonTablePath
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
index 8b87cfc..6cf1dcd 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
@@ -88,11 +89,13 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
child.initialize();
}
- private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) {
- String[] storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
- tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId,
- configuration.getSegmentId() + "", false, false);
+ private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) {
+ String[] storeLocation = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ tableIdentifier.getDatabaseName(),
+ tableIdentifier.getTableName(),
+ String.valueOf(configuration.getTaskNo()), configuration.getSegmentId(),
+ false,
+ false);
CarbonDataProcessorUtil.createLocations(storeLocation);
return storeLocation;
}
@@ -115,11 +118,11 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
measureCount = configuration.getMeasureCount();
outputLength = measureCount + (this.noDictWithComplextCount > 0 ? 1 : 0) + 1;
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
if (iterators.length == 1) {
- doExecute(iterators[0], 0, 0);
+ doExecute(iterators[0], 0);
} else {
executorService = Executors.newFixedThreadPool(iterators.length,
new CarbonThreadFactory("NoSortDataWriterPool:" + configuration.getTableIdentifier()
@@ -150,11 +153,10 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
return null;
}
- private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) {
- String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
- CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
- .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId,
- iteratorIndex);
+ private void doExecute(Iterator<CarbonRowBatch> iterator, int iteratorIndex) {
+ String[] storeLocation = getStoreLocation(tableIdentifier);
+ CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(
+ configuration, storeLocation, 0, iteratorIndex);
CarbonFactHandler dataHandler = null;
boolean rowsNotExist = true;
while (iterator.hasNext()) {
@@ -189,10 +191,11 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
processingComplete(dataHandler);
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis());
+ .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID,
+ System.currentTimeMillis());
}
private void processingComplete(CarbonFactHandler dataHandler) throws CarbonDataLoadingException {
@@ -298,7 +301,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
}
@Override public void run() {
- doExecute(this.iterator, 0, iteratorIndex);
+ doExecute(this.iterator, iteratorIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
index f030d52..369c1f2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
@@ -59,13 +60,11 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
child.initialize();
}
- private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) {
- String[] storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
- tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId,
- configuration.getSegmentId() + "", false, false);
- CarbonDataProcessorUtil.createLocations(storeLocation);
- return storeLocation;
+ private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) {
+ return CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ tableIdentifier.getDatabaseName(), tableIdentifier.getTableName(),
+ String.valueOf(configuration.getTaskNo()),
+ configuration.getSegmentId(), false, false);
}
@Override public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
@@ -75,18 +74,19 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
String tableName = tableIdentifier.getTableName();
try {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
int i = 0;
+ String[] storeLocation = getStoreLocation(tableIdentifier);
+ CarbonDataProcessorUtil.createLocations(storeLocation);
for (Iterator<CarbonRowBatch> iterator : iterators) {
- String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(i));
int k = 0;
while (iterator.hasNext()) {
CarbonRowBatch next = iterator.next();
// If no rows from merge sorter, then don't create a file in fact column handler
if (next.hasNext()) {
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
- .createCarbonFactDataHandlerModel(configuration, storeLocation, i, k++);
+ .createCarbonFactDataHandlerModel(configuration, storeLocation, 0, k++);
CarbonFactHandler dataHandler = CarbonFactHandlerFactory
.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
dataHandler.initialise();
@@ -119,10 +119,11 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
processingComplete(dataHandler);
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis());
+ .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID,
+ System.currentTimeMillis());
}
private void processingComplete(CarbonFactHandler dataHandler) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
index d321405..58009af 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
@@ -65,21 +66,21 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
child.initialize();
}
- private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) {
+ private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) {
String[] storeLocation = CarbonDataProcessorUtil
.getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
- tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId,
- configuration.getSegmentId() + "", false, false);
+ tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()),
+ configuration.getSegmentId(), false, false);
CarbonDataProcessorUtil.createLocations(storeLocation);
return storeLocation;
}
- public CarbonFactDataHandlerModel getDataHandlerModel(int partitionId) {
+ public CarbonFactDataHandlerModel getDataHandlerModel() {
CarbonTableIdentifier tableIdentifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
- String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
+ String[] storeLocation = getStoreLocation(tableIdentifier);
return CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration,
- storeLocation, partitionId, 0);
+ storeLocation, 0, 0);
}
@Override public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
@@ -89,11 +90,11 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
String tableName = tableIdentifier.getTableName();
try {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
int i = 0;
for (Iterator<CarbonRowBatch> iterator : iterators) {
- String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(i));
+ String[] storeLocation = getStoreLocation(tableIdentifier);
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
.createCarbonFactDataHandlerModel(configuration, storeLocation, i, 0);
@@ -147,10 +148,11 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
processingComplete(dataHandler);
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis());
+ .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID,
+ System.currentTimeMillis());
}
private void processingComplete(CarbonFactHandler dataHandler) throws CarbonDataLoadingException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index be27866..0eadc7f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -319,7 +319,7 @@ public final class CarbonDataMergerUtil {
// create entry for merged one.
LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails();
- loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId());
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID);
loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS);
long loadEnddate = CarbonUpdateUtil.readCurrentTime();
loadMetadataDetails.setLoadEndTime(loadEnddate);
@@ -676,7 +676,7 @@ public final class CarbonDataMergerUtil {
CarbonTableIdentifier carbonTableIdentifier, String segmentId) {
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(tablePath, carbonTableIdentifier);
- return carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ return carbonTablePath.getCarbonDataDirectoryPath(segmentId);
}
@@ -1036,7 +1036,7 @@ public final class CarbonDataMergerUtil {
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", seg);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(seg);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 2480a39..ff65db2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -372,7 +372,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
return SortParameters
.createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName,
dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount,
- noDictionaryCount, carbonLoadModel.getPartitionId(), segmentId,
+ noDictionaryCount, segmentId,
carbonLoadModel.getTaskNo(), noDictionaryColMapping, true);
}
@@ -422,7 +422,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
private void initTempStoreLocation() {
tempStoreLocation = CarbonDataProcessorUtil
.getLocalDataFolderLocation(carbonLoadModel.getDatabaseName(), tableName,
- carbonLoadModel.getTaskNo(), carbonLoadModel.getPartitionId(), segmentId,
+ carbonLoadModel.getTaskNo(), segmentId,
true, false);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
index a2248ee..0c0b2b0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -356,7 +357,7 @@ public class SortParameters implements Serializable {
CarbonProperties carbonProperties = CarbonProperties.getInstance();
parameters.setDatabaseName(tableIdentifier.getDatabaseName());
parameters.setTableName(tableIdentifier.getTableName());
- parameters.setPartitionID(configuration.getPartitionId());
+ parameters.setPartitionID("0");
parameters.setSegmentId(configuration.getSegmentId());
parameters.setTaskNo(configuration.getTaskNo());
parameters.setMeasureColCount(configuration.getMeasureCount());
@@ -392,10 +393,9 @@ public class SortParameters implements Serializable {
LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize());
- String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
- tableIdentifier.getTableName(), configuration.getTaskNo(),
- configuration.getPartitionId(), configuration.getSegmentId(), false, false);
+ String[] carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ tableIdentifier.getDatabaseName(), tableIdentifier.getTableName(),
+ configuration.getTaskNo(), configuration.getSegmentId(), false, false);
String[] sortTempDirs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
@@ -444,13 +444,13 @@ public class SortParameters implements Serializable {
public static SortParameters createSortParameters(CarbonTable carbonTable, String databaseName,
String tableName, int dimColCount, int complexDimColCount, int measureColCount,
- int noDictionaryCount, String partitionID, String segmentId, String taskNo,
+ int noDictionaryCount, String segmentId, String taskNo,
boolean[] noDictionaryColMaping, boolean isCompactionFlow) {
SortParameters parameters = new SortParameters();
CarbonProperties carbonProperties = CarbonProperties.getInstance();
parameters.setDatabaseName(databaseName);
parameters.setTableName(tableName);
- parameters.setPartitionID(partitionID);
+ parameters.setPartitionID(CarbonTablePath.DEPRECATED_PATITION_ID);
parameters.setSegmentId(segmentId);
parameters.setTaskNo(taskNo);
parameters.setMeasureColCount(measureColCount);
@@ -482,7 +482,7 @@ public class SortParameters implements Serializable {
LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize());
String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId,
+ .getLocalDataFolderLocation(databaseName, tableName, taskNo, segmentId,
isCompactionFlow, false);
String[] sortTempDirs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index d15152c..9f3c86f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -309,7 +309,7 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes);
String carbonDataDirectoryPath = CarbonDataProcessorUtil
.checkAndCreateCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(),
- tableName, loadModel.getPartitionId(), loadModel.getSegmentId());
+ tableName, loadModel.getSegmentId());
carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
List<CarbonDimension> dimensionByTableName = carbonTable.getDimensionByTableName(tableName);
boolean[] isUseInvertedIndexes = new boolean[dimensionByTableName.size()];
@@ -336,9 +336,8 @@ public class CarbonFactDataHandlerModel {
private static String getCarbonDataFolderLocation(CarbonDataLoadConfiguration configuration) {
AbsoluteTableIdentifier absoluteTableIdentifier = configuration.getTableIdentifier();
CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
- String carbonDataDirectoryPath = carbonTablePath
- .getCarbonDataDirectoryPath(configuration.getPartitionId(),
- configuration.getSegmentId() + "");
+ String carbonDataDirectoryPath =
+ carbonTablePath.getCarbonDataDirectoryPath(configuration.getSegmentId());
CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
return carbonDataDirectoryPath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 2a4cc00..cfe6e31 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -123,13 +123,11 @@ public final class CarbonDataProcessorUtil {
* @param databaseName
* @param tableName
* @param taskId
- * @param partitionId
* @param segmentId
* @return
*/
public static String[] getLocalDataFolderLocation(String databaseName, String tableName,
- String taskId, String partitionId, String segmentId, boolean isCompactionFlow,
- boolean isAltPartitionFlow) {
+ String taskId, String segmentId, boolean isCompactionFlow, boolean isAltPartitionFlow) {
String tempLocationKey =
getTempStoreLocationKey(databaseName, tableName, segmentId, taskId, isCompactionFlow,
isAltPartitionFlow);
@@ -150,8 +148,7 @@ public final class CarbonDataProcessorUtil {
String tmpStore = baseTmpStorePathArray[i];
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(tmpStore, carbonTable.getCarbonTableIdentifier());
- String carbonDataDirectoryPath =
- carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId + "");
+ String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
localDataFolderLocArray[i] = carbonDataDirectoryPath + File.separator + taskId;
}
@@ -378,13 +375,12 @@ public final class CarbonDataProcessorUtil {
* @return data directory path
*/
public static String checkAndCreateCarbonStoreLocation(String factStoreLocation,
- String databaseName, String tableName, String partitionId, String segmentId) {
+ String databaseName, String tableName, String segmentId) {
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName);
CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(factStoreLocation, carbonTableIdentifier);
- String carbonDataDirectoryPath =
- carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId);
+ String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath);
return carbonDataDirectoryPath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index fdc2cc3..f853319 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -77,10 +77,8 @@ public final class CarbonLoaderUtil {
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(loadModel.getTablePath(), carbonTable.getCarbonTableIdentifier());
- for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "", currentLoad + "");
- deleteStorePath(segmentPath);
- }
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(currentLoad + "");
+ deleteStorePath(segmentPath);
}
/**
@@ -100,7 +98,7 @@ public final class CarbonLoaderUtil {
int fileCount = 0;
int partitionCount = carbonTable.getPartitionCount();
for (int i = 0; i < partitionCount; i++) {
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "",
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(
currentLoad + "");
CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath,
FileFactory.getFileType(segmentPath));
@@ -274,7 +272,7 @@ public final class CarbonLoaderUtil {
private static void addToStaleFolders(CarbonTablePath carbonTablePath,
List<CarbonFile> staleFolders, LoadMetadataDetails entry) throws IOException {
- String path = carbonTablePath.getCarbonDataDirectoryPath("0", entry.getLoadName());
+ String path = carbonTablePath.getCarbonDataDirectoryPath(entry.getLoadName());
// add to the deletion list only if file exist else HDFS file system will throw
// exception while deleting the file if file path does not exist
if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
@@ -841,7 +839,7 @@ public final class CarbonLoaderUtil {
CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier();
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath(), carbonTableIdentifier);
- String segmentFolder = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentFolder = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonUtil.checkAndCreateFolder(segmentFolder);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
index 02ab1d8..f9f3e20 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
@@ -48,15 +48,14 @@ public final class DeleteLoadFolders {
* returns segment path
*
* @param absoluteTableIdentifier
- * @param partitionId
* @param oneLoad
* @return
*/
private static String getSegmentPath(AbsoluteTableIdentifier absoluteTableIdentifier,
- int partitionId, LoadMetadataDetails oneLoad) {
+ LoadMetadataDetails oneLoad) {
CarbonTablePath carbon = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier);
String segmentId = oneLoad.getLoadName();
- return carbon.getCarbonDataDirectoryPath("" + partitionId, segmentId);
+ return carbon.getCarbonDataDirectoryPath(segmentId);
}
public static void physicalFactAndMeasureMetadataDeletion(
@@ -64,7 +63,7 @@ public final class DeleteLoadFolders {
LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
for (LoadMetadataDetails oneLoad : currentDetails) {
if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) {
- String path = getSegmentPath(absoluteTableIdentifier, 0, oneLoad);
+ String path = getSegmentPath(absoluteTableIdentifier, oneLoad);
boolean status = false;
try {
if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
index e662757..7f0aef6 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
@@ -163,7 +163,6 @@ public class StoreCreator {
loadModel.setCsvHeaderColumns(loadModel.getCsvHeader().split(","));
loadModel.setTaskNo("0");
loadModel.setSegmentId("0");
- loadModel.setPartitionId("0");
loadModel.setFactTimeStamp(System.currentTimeMillis());
loadModel.setMaxColumns("10");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index a96ab32..36a5a15 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -105,7 +105,6 @@ class StreamHandoffRDD[K, V](
split: Partition,
context: TaskContext
): Iterator[(K, V)] = {
- carbonLoadModel.setPartitionId("0")
carbonLoadModel.setTaskNo("" + split.index)
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
CarbonMetadata.getInstance().addCarbonTable(carbonTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index 1c7be6a..f2274be 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -135,7 +135,7 @@ object StreamSinkFactory {
FileFactory.mkdirs(carbonTablePath.getMetadataDirectoryPath, fileType)
}
val segmentId = StreamSegment.open(carbonTable)
- val segmentDir = carbonTablePath.getSegmentDir("0", segmentId)
+ val segmentDir = carbonTablePath.getSegmentDir(segmentId)
if (FileFactory.isFileExist(segmentDir, fileType)) {
// recover fault
StreamSegment.recoverSegmentIfRequired(segmentDir)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index 849bf99..45bc19a 100644
--- a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++ b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -149,12 +149,12 @@ class CarbonAppendableStreamSink(
* if the directory size of current segment beyond the threshold, hand off new segment
*/
private def checkOrHandOffSegment(): Unit = {
- val segmentDir = carbonTablePath.getSegmentDir("0", currentSegmentId)
+ val segmentDir = carbonTablePath.getSegmentDir(currentSegmentId)
val fileType = FileFactory.getFileType(segmentDir)
if (segmentMaxSize <= StreamSegment.size(segmentDir)) {
val newSegmentId = StreamSegment.close(carbonTable, currentSegmentId)
currentSegmentId = newSegmentId
- val newSegmentDir = carbonTablePath.getSegmentDir("0", currentSegmentId)
+ val newSegmentDir = carbonTablePath.getSegmentDir(currentSegmentId)
FileFactory.mkdirs(newSegmentDir, fileType)
// TODO trigger hand off operation
@@ -251,14 +251,14 @@ object CarbonAppendableStreamSink {
// update data file info in index file
val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- StreamSegment.updateIndexFile(tablePath.getSegmentDir("0", segmentId))
+ StreamSegment.updateIndexFile(tablePath.getSegmentDir(segmentId))
} catch {
// catch fault of executor side
case t: Throwable =>
val tablePath =
CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = tablePath.getSegmentDir("0", segmentId)
+ val segmentDir = tablePath.getSegmentDir(segmentId)
StreamSegment.recoverSegmentIfRequired(segmentDir)
LOGGER.error(t, s"Aborting job ${ job.getJobID }.")
committer.abortJob(job)
[25/50] [abbrv] carbondata git commit: [CARBONDATA-2089] Test cases
is incorrect because it always run success no matter whether the SQL thrown
exception
Posted by ja...@apache.org.
[CARBONDATA-2089] Test cases is incorrect because it always run success no matter whether the SQL thrown exception
exception is error. Because some test case always run success whether sql throw exception
This closes #1866
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/842b9e5f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/842b9e5f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/842b9e5f
Branch: refs/heads/carbonstore
Commit: 842b9e5f4bf524c08b3ad4f17d93d45a21cb1912
Parents: 2e6bc66
Author: xubo245 <60...@qq.com>
Authored: Fri Jan 26 20:40:20 2018 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Sun Jan 28 15:14:01 2018 +0800
----------------------------------------------------------------------
.../dataload/TestLoadDataGeneral.scala | 10 +--
.../preaggregate/TestPreAggCreateCommand.scala | 41 +++++----
.../timeseries/TestTimeSeriesCreateTable.scala | 88 +++++++++++++-------
.../InsertIntoCarbonTableTestCase.scala | 9 +-
.../MajorCompactionIgnoreInMinorTest.scala | 6 +-
.../TestDataLoadWithColumnsMoreThanSchema.scala | 37 +++++---
.../TestLoadDataWithDiffTimestampFormat.scala | 56 +++++--------
.../TestLoadDataWithFileHeaderException.scala | 76 +++++++----------
...ataWithMalformedCarbonCommandException.scala | 52 ++++--------
.../TestLoadDataWithNotProperInputFile.scala | 16 ++--
.../dataretention/DataRetentionTestCase.scala | 75 ++++-------------
.../deleteTable/TestDeleteTableNewDDL.scala | 10 +--
.../ColumnPropertyValidationTestCase.scala | 21 +++--
.../util/ExternalColumnDictionaryTestCase.scala | 24 ++----
.../commands/SetCommandTestCase.scala | 42 ++--------
.../register/TestRegisterCarbonTable.scala | 6 +-
16 files changed, 232 insertions(+), 337 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index 0be0b6d..c122287 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -175,17 +175,11 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
sql("CREATE TABLE load32000chardata(dim1 String, dim2 String, mes1 int) STORED BY 'org.apache.carbondata.format'")
sql("CREATE TABLE load32000chardata_dup(dim1 String, dim2 String, mes1 int) STORED BY 'org.apache.carbondata.format'")
sql(s"LOAD DATA LOCAL INPATH '$testdata' into table load32000chardata OPTIONS('FILEHEADER'='dim1,dim2,mes1')")
- try{
+ intercept[Exception] {
sql("insert into load32000chardata_dup select dim1,concat(load32000chardata.dim2,'aaaa'),mes1 from load32000chardata").show()
- assert(false)
- } catch {
- case _:Exception => assert(true)
}
- try{
+ intercept[Exception] {
sql("update load32000chardata_dup set(load32000chardata_dup.dim2)=(select concat(load32000chardata.dim2,'aaaa') from load32000chardata)").show()
- assert(false)
- } catch {
- case _:Exception => assert(true)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index 5784bf2..755a449 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -118,35 +118,40 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
}
test("test pre agg create table 13") {
- try {
+ intercept[Exception] {
sql(
- "create datamap preagg19 on table PreAggMain2 using 'preaggregate' as select a as a1,count(distinct b) from PreAggMain2 group by a")
- assert(false)
- } catch {
- case _: Exception =>
- assert(true)
+ s"""
+ | create datamap preagg19 on table PreAggMain2
+ | using 'preaggregate'
+ | as select a as a1,count(distinct b)
+ | from PreAggMain2 group by a
+ """.stripMargin)
}
}
test("test pre agg create table 14") {
- try {
+ intercept[Exception] {
sql(
- "create datamap preagg20 on table PreAggMain2 using 'preaggregate' as select a as a1,sum(distinct b) from PreAggMain2 group by a")
- assert(false)
- } catch {
- case _: Exception =>
- assert(true)
+ s"""
+ | create datamap preagg20 on table PreAggMain2
+ | using 'preaggregate'
+ | as select a as a1,sum(distinct b) from PreAggMain2
+ | group by a
+ """.stripMargin)
}
}
test("test pre agg create table 15") {
- try {
+ intercept[Exception] {
sql(
- "create datamap preagg21 on table PreAggMain2 using 'preaggregate' as select a as a1,sum(b) from PreAggMain2 where a='vishal' group by a")
- assert(false)
- } catch {
- case _: Exception =>
- assert(true)
+ s"""
+ | create datamap preagg21 on table PreAggMain2
+ | using 'preaggregate'
+ | as select a as a1,sum(b)
+ | from PreAggMain2
+ | where a='vishal'
+ | group by a
+ """.stripMargin)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
index 5cbcb26..c9041fa 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
@@ -50,59 +50,87 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
}
test("test timeseries create table five") {
- try {
+ intercept[Exception] {
sql(
- "create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='sec=1,hour=1,day=1,month=1,year=1') as select dataTime, sum(age) from mainTable group by dataTime")
- assert(false)
- } catch {
- case _:Exception =>
- assert(true)
+ s"""
+ | create datamap agg0 on table mainTable
+ | using 'preaggregate'
+ | DMPROPERTIES (
+ | 'timeseries.eventTime'='dataTime',
+ | 'timeseries.hierarchy'='sec=1,hour=1,day=1,month=1,year=1')
+ | as select dataTime, sum(age) from mainTable
+ | group by dataTime
+ """.stripMargin)
}
}
test("test timeseries create table Six") {
- try {
+ intercept[Exception] {
sql(
- "create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='hour=2') as select dataTime, sum(age) from mainTable group by dataTime")
- assert(false)
- } catch {
- case _:Exception =>
- assert(true)
+ """
+ | create datamap agg0 on table mainTable
+ | using 'preaggregate'
+ | DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='hour=2')
+ | as select dataTime, sum(age) from mainTable
+ | group by dataTime
+ """.stripMargin)
+
}
}
test("test timeseries create table seven") {
- try {
+ intercept[Exception] {
+ sql(
+ s"""
+ | create datamap agg0 on table mainTable
+ | using 'preaggregate'
+ | DMPROPERTIES (
+ | 'timeseries.eventTime'='dataTime',
+ | 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
+ | as select dataTime, sum(age) from mainTable
+ | group by dataTime
+ """.stripMargin)
sql(
- "create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1') as select dataTime, sum(age) from mainTable group by dataTime")
- assert(false)
- } catch {
- case _:Exception =>
- assert(true)
+ s"""
+ | create datamap agg0 on table mainTable
+ | using 'preaggregate'
+ | DMPROPERTIES (
+ | 'timeseries.eventTime'='dataTime',
+ | 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
+ | as select dataTime, sum(age) from mainTable
+ | group by dataTime
+ """.stripMargin)
}
}
test("test timeseries create table Eight") {
- try {
+ intercept[Exception] {
sql(
- "create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='name', 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1') as select name, sum(age) from mainTable group by name")
- assert(false)
- } catch {
- case _:Exception =>
- assert(true)
+ s"""
+ | create datamap agg0 on table mainTable
+ | using 'preaggregate'
+ | DMPROPERTIES ('timeseries.eventTime'='name', 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
+ | as select name, sum(age) from mainTable
+ | group by name
+ """.stripMargin)
}
}
test("test timeseries create table Nine") {
- try {
+ intercept[Exception] {
sql(
- "create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1') as select name, sum(age) from mainTable group by name")
- assert(false)
- } catch {
- case _:Exception =>
- assert(true)
+ s"""
+ | create datamap agg0 on table mainTable
+ | using 'preaggregate'
+ | DMPROPERTIES (
+ | 'timeseries.eventTime'='dataTime',
+ | 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
+ | as select name, sum(age) from mainTable
+ | group by name
+ """.stripMargin)
}
}
+
override def afterAll: Unit = {
sql("drop table if exists mainTable")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index 6739c6c..d59f0b5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -88,12 +88,9 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
test("insert->carbon column is more then hive-fails") {
sql("drop table if exists TCarbon")
sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
- try {
- sql("insert into TCarbon select imei,deviceInformationId,MAC,deviceColor,gamePointId from THive")
- assert(false)
- } catch {
- case ex: Exception => assert(true)
- }
+ intercept[Exception] {
+ sql("insert into TCarbon select imei,deviceInformationId,MAC,deviceColor,gamePointId from THive")
+ }
}
test("insert->insert wrong data types-pass") {
sql("drop table if exists TCarbon")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index 6155d7c..86f974c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -104,12 +104,8 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
* Delete should not work on compacted segment.
*/
test("delete compacted segment and check status") {
- try {
+ intercept[Throwable] {
sql("delete from table ignoremajor where segment.id in (2)")
- assert(false)
- }
- catch {
- case _:Throwable => assert(true)
}
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
index 1e34ec8..1532328 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
@@ -48,12 +48,26 @@ class TestDataLoadWithColumnsMoreThanSchema extends QueryTest with BeforeAndAfte
test("test for invalid value of maxColumns") {
sql("DROP TABLE IF EXISTS max_columns_test")
- sql("CREATE TABLE max_columns_test (imei string,age int,task bigint,num double,level decimal(10,3),productdate timestamp,mark int,name string)STORED BY 'org.apache.carbondata.format'")
- try {
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/character_carbon.csv' into table max_columns_test options('MAXCOLUMNS'='avfgd')")
- assert(false)
- } catch {
- case _: Throwable => assert(true)
+ sql(
+ s"""
+ | CREATE TABLE max_columns_test (
+ | imei string,
+ | age int,
+ | task bigint,
+ | num double,
+ | level decimal(10,3),
+ | productdate timestamp,
+ | mark int,
+ | name string)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ intercept[Throwable] {
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$resourcesPath/character_carbon.csv'
+ | into table max_columns_test
+ | options('MAXCOLUMNS'='avfgd')
+ """.stripMargin)
}
}
@@ -74,15 +88,12 @@ class TestDataLoadWithColumnsMoreThanSchema extends QueryTest with BeforeAndAfte
"CREATE TABLE max_columns_value_test (imei string,age int,task bigint,num double,level " +
"decimal(10,3),productdate timestamp,mark int,name string) STORED BY 'org.apache.carbondata" +
".format'")
- try {
+
+
+ intercept[Throwable] {
sql(
s"LOAD DATA LOCAL INPATH '$resourcesPath/character_carbon.csv' into table " +
- "max_columns_value_test options('FILEHEADER='imei,age','MAXCOLUMNS'='2')")
- throw new MalformedCarbonCommandException("Invalid")
- } catch {
- case me: MalformedCarbonCommandException =>
- assert(false)
- case _: Throwable => assert(true)
+ "max_columns_value_test options('FILEHEADER='imei,age','MAXCOLUMNS'='2')")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
index 1b0da78..ec6fff1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
@@ -76,65 +76,51 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
}
test("test load data with different timestamp format with wrong setting") {
- try {
- sql(s"""
+
+ val ex = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
OPTIONS('dateformat' = 'date')
""")
- assert(false)
- } catch {
- case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Wrong option: date is provided for option DateFormat")
- case _: Throwable=> assert(false)
}
+ assertResult(ex.getMessage)("Error: Wrong option: date is provided for option DateFormat")
- try {
- sql(s"""
+ val ex0 = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
OPTIONS('timestampformat' = 'timestamp')
""")
- assert(false)
- } catch {
- case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Wrong option: timestamp is provided for option TimestampFormat")
- case _: Throwable => assert(false)
}
+ assertResult(ex0.getMessage)("Error: Wrong option: timestamp is provided for option TimestampFormat")
- try {
- sql(s"""
+ val ex1 = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
OPTIONS('dateformat' = 'date: ')
""")
- assert(false)
- } catch {
- case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Wrong option: date: is provided for option DateFormat")
- case _: Throwable => assert(false)
}
+ assertResult(ex1.getMessage)("Error: Wrong option: date: is provided for option DateFormat")
- try {
- sql(s"""
+ val ex2 = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
OPTIONS('dateformat' = 'date ')
""")
- assert(false)
- } catch {
- case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Wrong option: date is provided for option DateFormat")
- case _: Throwable => assert(false)
}
+ assertResult(ex2.getMessage)("Error: Wrong option: date is provided for option DateFormat")
- try {
- sql(s"""
+ val ex3 = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
OPTIONS('dateformat' = 'fasfdas:yyyy/MM/dd')
""")
- assert(false)
- } catch {
- case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Wrong option: fasfdas:yyyy/MM/dd is provided for option DateFormat")
- case _: Throwable => assert(false)
}
+ assertResult(ex3.getMessage)("Error: Wrong option: fasfdas:yyyy/MM/dd is provided for option DateFormat")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
index 3d3fb0c..7700ed5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
@@ -32,54 +32,45 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
}
test("test load data both file and ddl without file header exception") {
- try {
- sql(s"""
- LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
- """)
- assert(false)
- } catch {
- case e: Exception =>
- assert(e.getMessage.contains("CSV header in input file is not proper. Column names in schema and csv header are not the same."))
+ val e = intercept[Exception] {
+ sql(
+ s"""LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3""")
}
+ assert(e.getMessage.contains(
+ "CSV header in input file is not proper. Column names in schema and csv header are not the same."))
}
test("test load data ddl provided wrong file header exception") {
- try {
- sql(s"""
+ val e = intercept[Exception] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
options('fileheader'='no_column')
""")
- assert(false)
- } catch {
- case e: Exception =>
- assert(e.getMessage.contains("CSV header in DDL is not proper. Column names in schema and CSV header are not the same"))
}
+ assert(e.getMessage.contains("CSV header in DDL is not proper. Column names in schema and CSV header are not the same"))
}
test("test load data with wrong header , but without fileheader") {
- try {
- sql(s"""
+ val e = intercept[Exception] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source.csv' into table t3
options('header'='abc')
""")
- assert(false)
- } catch {
- case e: Exception =>
- assert(e.getMessage.contains("'header' option should be either 'true' or 'false'"))
}
+ assert(e.getMessage.contains("'header' option should be either 'true' or 'false'"))
}
test("test load data with wrong header and fileheader") {
- try {
- sql(s"""
+ val e = intercept[Exception] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
options('header'='', 'fileheader'='ID,date,country,name,phonetype,serialname,salary')
""")
- assert(false)
- } catch {
- case e: Exception =>
- assert(e.getMessage.contains("'header' option should be either 'true' or 'false'"))
}
+ assert(e.getMessage.contains("'header' option should be either 'true' or 'false'"))
}
test("test load data with header=false, but without fileheader") {
@@ -97,16 +88,14 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
}
test("test load data with header=false and wrong fileheader") {
- try {
- sql(s"""
+ val e = intercept[Exception] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
options('header'='false', 'fileheader'='ID1,date2,country,name,phonetype,serialname,salary')
""")
- assert(false)
- } catch {
- case e: Exception =>
- assert(e.getMessage.contains("CSV header in DDL is not proper. Column names in schema and CSV header are not the same"))
}
+ assert(e.getMessage.contains("CSV header in DDL is not proper. Column names in schema and CSV header are not the same"))
}
test("test load data with header=true, but without fileheader") {
@@ -117,29 +106,27 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
}
test("test load data with header=true and fileheader") {
- try {
- sql(s"""
+
+ val e = intercept[Exception] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source.csv' into table t3
options('header'='true', 'fileheader'='ID,date,country,name,phonetype,serialname,salary')
""")
- assert(false)
- } catch {
- case e: Exception =>
- assert(e.getMessage.contains("When 'header' option is true, 'fileheader' option is not required."))
}
+ assert(e.getMessage.contains("When 'header' option is true, 'fileheader' option is not required."))
}
test("test load data with header=true and wrong fileheader") {
- try {
- sql(s"""
+
+ val e = intercept[Exception] {
+ sql(
+ s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source.csv' into table t3
options('header'='true', 'fileheader'='ID1,date1,country,name,phonetype,serialname,salary')
""")
- assert(false)
- } catch {
- case e: Exception =>
- assert(e.getMessage.contains("When 'header' option is true, 'fileheader' option is not required."))
}
+ assert(e.getMessage.contains("When 'header' option is true, 'fileheader' option is not required."))
}
test("test load data without header and fileheader") {
@@ -155,6 +142,7 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
""")
}
+
override def afterAll {
sql("DROP TABLE IF EXISTS t3")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
index 3462a07..1851705 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
@@ -79,60 +79,43 @@ class TestLoadDataWithMalformedCarbonCommandException extends QueryTest with Bef
}
test("test load data with dictionary exclude columns which no exist in table.") {
- try {
+ val e = intercept[MalformedCarbonCommandException] {
buildTableWithNoExistDictExclude()
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(e.getMessage.equals("DICTIONARY_EXCLUDE column: ccc does not exist in table. " +
- "Please check create table statement."))
- case _: Throwable => assert(false)
}
+ assert(e.getMessage.equals("DICTIONARY_EXCLUDE column: ccc does not exist in table. " +
+ "Please check create table statement."))
}
test("test load data with dictionary include columns which no exist in table.") {
- try {
+ val e = intercept[MalformedCarbonCommandException] {
buildTableWithNoExistDictInclude()
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(e.getMessage.equals("DICTIONARY_INCLUDE column: aaa does not exist in table. " +
- "Please check create table statement."))
- case _: Throwable => assert(false)
}
+ assert(e.getMessage.equals("DICTIONARY_INCLUDE column: aaa does not exist in table. " +
+ "Please check create table statement."))
}
test("test load data with dictionary include is same with dictionary exclude") {
- try {
+ val e = intercept[MalformedCarbonCommandException] {
buildTableWithSameDictExcludeAndInclude()
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(e.getMessage.equals("DICTIONARY_EXCLUDE can not contain the same column: country " +
- "with DICTIONARY_INCLUDE. Please check create table statement."))
- case _: Throwable => assert(false)
}
+ assert(e.getMessage.equals("DICTIONARY_EXCLUDE can not contain the same column: country " +
+ "with DICTIONARY_INCLUDE. Please check create table statement."))
}
test("test load data with invalid option") {
- try {
+ val e = intercept[MalformedCarbonCommandException] {
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE " +
"TestLoadTableOptions OPTIONS('QUOTECHAR'='\"', 'DELIMITERRR' = ',')")
- assert(false)
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(e.getMessage.equals("Error: Invalid option(s): delimiterrr"))
- case _: Throwable => assert(false)
}
+ assert(e.getMessage.equals("Error: Invalid option(s): delimiterrr"))
}
test("test load data with duplicate options") {
- try {
+ val e = intercept[MalformedCarbonCommandException] {
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE " +
"TestLoadTableOptions OPTIONS('DELIMITER' = ',', 'quotechar'='\"', 'DELIMITER' = '$')")
- assert(false)
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(e.getMessage.equals("Error: Duplicate option(s): delimiter"))
- case _: Throwable => assert(false)
}
+ assert(e.getMessage.equals("Error: Duplicate option(s): delimiter"))
}
test("test load data with case sensitive options") {
@@ -147,13 +130,10 @@ class TestLoadDataWithMalformedCarbonCommandException extends QueryTest with Bef
}
test("test load data with dictionary include is same with dictionary exclude with spaces") {
- try {
+ val e = intercept[MalformedCarbonCommandException] {
buildTableWithSameDictExcludeAndIncludeWithSpaces()
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(e.getMessage.equals("DICTIONARY_EXCLUDE can not contain the same column: country " +
- "with DICTIONARY_INCLUDE. Please check create table statement."))
- case _: Throwable => assert(false)
}
+ assert(e.getMessage.equals("DICTIONARY_EXCLUDE can not contain the same column: country " +
+ "with DICTIONARY_INCLUDE. Please check create table statement."))
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
index 47f05ce..a4d7b85 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
@@ -27,15 +27,12 @@ import org.apache.spark.sql.test.util.QueryTest
class TestLoadDataWithNotProperInputFile extends QueryTest {
test("test loading data with input path exists but has nothing") {
- try {
+ val e = intercept[Throwable] {
val dataPath = s"$resourcesPath/nullSample.csv"
FileUtils.getPaths(dataPath)
- assert(false)
- } catch {
- case e: Throwable =>
- assert(e.getMessage.contains("Please check your input path and make sure " +
- "that files end with '.csv' and content is not empty"))
}
+ assert(e.getMessage.contains("Please check your input path and make sure " +
+ "that files end with '.csv' and content is not empty"))
}
test("test loading data with input file not ends with '.csv'") {
@@ -50,13 +47,10 @@ class TestLoadDataWithNotProperInputFile extends QueryTest {
}
test("test loading data with input file does not exist") {
- try {
+ val e = intercept[Throwable] {
val dataPath = s"$resourcesPath/input_file_does_not_exist.csv"
FileUtils.getPaths(dataPath)
- assert(false)
- } catch {
- case e: Throwable =>
- assert(e.getMessage.contains("The input file does not exist"))
}
+ assert(e.getMessage.contains("The input file does not exist"))
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
index bde1f80..ed58253 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
@@ -173,15 +173,11 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
}
test("RetentionTest4_DeleteByInvalidLoadId") {
- try {
+ val e = intercept[MalformedCarbonCommandException] {
// delete segment with no id
sql("delete from table DataRetentionTable where segment.id in ()")
- assert(false)
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(e.getMessage.contains("should not be empty"))
- case _: Throwable => assert(false)
}
+ assert(e.getMessage.contains("should not be empty"))
}
test("test delete segments by load date with case-insensitive table name") {
@@ -211,27 +207,19 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
test("RetentionTest_DeleteSegmentsByLoadTimeValiadtion") {
- try {
+ val e = intercept[MalformedCarbonCommandException] {
sql(
"delete from table DataRetentionTable where segment.starttime before" +
- " 'abcd-01-01 00:00:00'")
- assert(false)
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(e.getMessage.contains("Invalid load start time format"))
- case _: Throwable => assert(false)
+ " 'abcd-01-01 00:00:00'")
}
+ assert(e.getMessage.contains("Invalid load start time format"))
- try {
+ val ex = intercept[MalformedCarbonCommandException] {
sql(
"delete from table DataRetentionTable where segment.starttime before" +
- " '2099:01:01 00:00:00'")
- assert(false)
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(e.getMessage.contains("Invalid load start time format"))
- case _: Throwable => assert(false)
+ " '2099:01:01 00:00:00'")
}
+ assert(ex.getMessage.contains("Invalid load start time format"))
checkAnswer(
sql("SELECT country, count(salary) AS amount FROM DataRetentionTable WHERE country" +
@@ -249,31 +237,16 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
test("RetentionTest_InvalidDeleteCommands") {
// All these queries should fail.
- try {
+ intercept[Exception] {
sql("DELETE LOADS FROM TABLE DataRetentionTable where STARTTIME before '2099-01-01'")
- throw new MalformedCarbonCommandException("Invalid query")
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(!e.getMessage.equalsIgnoreCase("Invalid query"))
- case _: Throwable => assert(true)
}
- try {
+ intercept[Exception] {
sql("DELETE LOAD 2 FROM TABLE DataRetentionTable")
- throw new MalformedCarbonCommandException("Invalid query")
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(!e.getMessage.equalsIgnoreCase("Invalid query"))
- case _: Throwable => assert(true)
}
- try {
+ intercept[Exception] {
sql("show loads for table DataRetentionTable")
- throw new MalformedCarbonCommandException("Invalid query")
- } catch {
- case e: MalformedCarbonCommandException =>
- assert(!e.getMessage.equalsIgnoreCase("Invalid query"))
- case _: Throwable => assert(true)
}
}
@@ -288,37 +261,19 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
carbonCleanFilesLock.lockWithRetries()
// delete segment 0 it should fail
- try {
+ intercept[Exception] {
sql("delete from table retentionlock where segment.id in (0)")
- throw new MalformedCarbonCommandException("Invalid")
- } catch {
- case me: MalformedCarbonCommandException =>
- assert(false)
- case ex: Exception =>
- assert(true)
}
// it should fail
- try {
+ intercept[Exception] {
sql("delete from table retentionlock where segment.starttime before " +
- "'2099-01-01 00:00:00.0'")
- throw new MalformedCarbonCommandException("Invalid")
- } catch {
- case me: MalformedCarbonCommandException =>
- assert(false)
- case ex: Exception =>
- assert(true)
+ "'2099-01-01 00:00:00.0'")
}
// it should fail
- try {
+ intercept[Exception] {
sql("clean files for table retentionlock")
- throw new MalformedCarbonCommandException("Invalid")
- } catch {
- case me: MalformedCarbonCommandException =>
- assert(false)
- case ex: Exception =>
- assert(true)
}
sql("SHOW SEGMENTS FOR TABLE retentionlock").show
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/deleteTable/TestDeleteTableNewDDL.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/deleteTable/TestDeleteTableNewDDL.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/deleteTable/TestDeleteTableNewDDL.scala
index 2f30215..811cb02 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/deleteTable/TestDeleteTableNewDDL.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/deleteTable/TestDeleteTableNewDDL.scala
@@ -64,18 +64,12 @@ class TestDeleteTableNewDDL extends QueryTest with BeforeAndAfterAll {
sql("use testdb")
sql("CREATE TABLE IF NOT EXISTS testtable(empno Int, empname string, utilization Int,salary Int)"
+ " STORED BY 'org.apache.carbondata.format' ")
- try {
+ intercept[Exception] {
sql("drop database testdb")
- assert(false)
- } catch {
- case _ : Exception =>
}
sql("drop database testdb cascade")
- try {
+ intercept[Exception] {
sql("use testdb")
- assert(false)
- } catch {
- case _ : Exception =>
}
sql("use default")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnPropertyValidationTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnPropertyValidationTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnPropertyValidationTestCase.scala
index b3ea677..69cccf7 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnPropertyValidationTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnPropertyValidationTestCase.scala
@@ -34,13 +34,20 @@ class ColumnPropertyValidationTestCase extends QueryTest with BeforeAndAfterAll
}
}
test("Validate Dictionary include _ invalid key") {
- try {
- sql("create table employee(empname String,empid String,city String,country String,gender String,salary Double) stored by 'org.apache.carbondata.format' tblproperties('columnproperties.invalid.key'='value')")
- assert(false)
- sql("drop table employee")
- } catch {
- case e: Throwable =>assert(true)
- }
+ intercept[Throwable] {
+ sql(
+ s"""
+ | create table employee(
+ | empname String,
+ | empid String,
+ | city String,
+ | country String,
+ | gender String,
+ | salary Double)
+ | stored by 'org.apache.carbondata.format'
+ | tblproperties('columnproperties.invalid.key'='value')
+ """.stripMargin)
+ }
}
override def afterAll() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index cad7807..a1b39d8 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -263,37 +263,29 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
}
test("COLUMNDICT and ALL_DICTIONARY_PATH can not be used together") {
- try {
+ val ex = intercept[MalformedCarbonCommandException] {
sql(
s"""
LOAD DATA LOCAL INPATH "$complexFilePath1" INTO TABLE loadSqlTest
OPTIONS('COLUMNDICT'='$extColDictFilePath1',"ALL_DICTIONARY_PATH"='$extColDictFilePath1')
""")
- assert(false)
- } catch {
- case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)(
- "Error: COLUMNDICT and ALL_DICTIONARY_PATH can not be used together " +
- "in options")
- case _: Throwable => assert(false)
}
+ assertResult(ex.getMessage)(
+ "Error: COLUMNDICT and ALL_DICTIONARY_PATH can not be used together " +
+ "in options")
}
test("Measure can not use COLUMNDICT") {
- try {
+ val ex = intercept[DataLoadingException] {
sql(
s"""
LOAD DATA LOCAL INPATH "$complexFilePath1" INTO TABLE loadSqlTest
OPTIONS('single_pass'='true','FILEHEADER'='$header', 'COLUMNDICT'='gamePointId:$filePath')
""")
- assert(false)
- } catch {
- case ex: DataLoadingException =>
- assertResult(ex.getMessage)(
- "Column gamePointId is not a key column. Only key column can be part " +
- "of dictionary and used in COLUMNDICT option.")
- case _: Throwable => assert(false)
}
+ assertResult(ex.getMessage)(
+ "Column gamePointId is not a key column. Only key column can be part " +
+ "of dictionary and used in COLUMNDICT option.")
}
def cleanAllTables: Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala
index 60cf121..303a478 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala
@@ -39,12 +39,8 @@ class SetCommandTestCase extends Spark2QueryTest with BeforeAndAfterAll{
}
test("test set command for enable.unsafe.sort for invalid option") {
- try {
+ intercept[InvalidConfigurationException] {
checkAnswer(sql("set enable.unsafe.sort=123"), sql("set enable.unsafe.sort"))
- assert(false)
- } catch {
- case ex: InvalidConfigurationException =>
- assert(true)
}
}
//is_empty_data_bad_record
@@ -58,15 +54,11 @@ class SetCommandTestCase extends Spark2QueryTest with BeforeAndAfterAll{
test(s"test set command for ${
CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE} for invalid option") {
- try {
+ intercept[InvalidConfigurationException] {
checkAnswer(sql(s"set ${
CarbonLoadOptionConstants
.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE
}=123"), sql(s"set ${ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE }"))
- assert(false)
- } catch {
- case ex: InvalidConfigurationException =>
- assert(true)
}
}
test(s"test set command for ${
@@ -82,14 +74,10 @@ class SetCommandTestCase extends Spark2QueryTest with BeforeAndAfterAll{
test(s"test set command for ${CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD} " +
s"for invalid option") {
- try {
+ intercept[InvalidConfigurationException] {
checkAnswer(
sql(s"set ${CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD}=123"),
sql(s"set ${CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD}"))
- assert(false)
- } catch {
- case ex: InvalidConfigurationException =>
- assert(true)
}
}
//carbon.custom.block.distribution
@@ -99,13 +87,9 @@ class SetCommandTestCase extends Spark2QueryTest with BeforeAndAfterAll{
}
test("test set command for carbon.custom.block.distribution for invalid option") {
- try {
+ intercept[InvalidConfigurationException] {
checkAnswer(sql("set carbon.custom.block.distribution=123"),
sql("set carbon.custom.block.distribution"))
- assert(false)
- } catch {
- case ex: InvalidConfigurationException =>
- assert(true)
}
}
// sort_scope
@@ -115,13 +99,9 @@ class SetCommandTestCase extends Spark2QueryTest with BeforeAndAfterAll{
}
test(s"test set command for ${CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE} for invalid option") {
- try {
+ intercept[InvalidConfigurationException] {
checkAnswer(sql(s"set ${CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE}=123"),
sql(s"set ${CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE}"))
- assert(false)
- } catch {
- case ex: InvalidConfigurationException =>
- assert(true)
}
}
// batch_sort_size_inmb
@@ -131,13 +111,9 @@ class SetCommandTestCase extends Spark2QueryTest with BeforeAndAfterAll{
}
test(s"test set ${CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB} for invalid option") {
- try {
+ intercept[InvalidConfigurationException] {
checkAnswer(sql(s"set ${CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB}=hjf"),
sql(s"set ${CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB}"))
- assert(false)
- } catch {
- case ex: InvalidConfigurationException =>
- assert(true)
}
}
// single_pass
@@ -147,13 +123,9 @@ class SetCommandTestCase extends Spark2QueryTest with BeforeAndAfterAll{
}
test(s"test set ${CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS} for invalid option") {
- try {
+ intercept[InvalidConfigurationException] {
checkAnswer(sql(s"set ${CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS}=123"),
sql(s"set ${CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS}"))
- assert(false)
- } catch {
- case ex: InvalidConfigurationException =>
- assert(true)
}
}
override def afterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/842b9e5f/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala
index 1b36e17..389f2cd 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/register/TestRegisterCarbonTable.scala
@@ -150,12 +150,8 @@ class TestRegisterCarbonTable extends QueryTest with BeforeAndAfterAll {
sql("drop table carbontable")
if (!CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.isReadFromHiveMetaStore) {
restoreData(dblocation, "carbontable")
- try {
+ intercept[AnalysisException] {
sql("refresh table carbontable")
- assert(false)
- } catch {
- case e: AnalysisException =>
- assert(true)
}
restoreData(dblocation, "carbontable_preagg1")
}
[40/50] [abbrv] carbondata git commit: [CARBONDATA-2049]
CarbonCleanFilesCommand table path problem
Posted by ja...@apache.org.
[CARBONDATA-2049] CarbonCleanFilesCommand table path problem
Problem:
In CarbonCleanFilesCommand datbaseLocation is being passed instead of the tablePath in case of forceclean.
And in case of cleanGarbageData, storeLocation is being passed instead of the tablePath.
This closes #1828
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9b479617
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9b479617
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9b479617
Branch: refs/heads/carbonstore
Commit: 9b4796177610e3a4f9d426169753a40eceb7b675
Parents: d509f17
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Tue Jan 16 11:49:54 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 09:37:39 2018 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/api/CarbonStore.scala | 32 ++++++++++++++------
.../management/CarbonCleanFilesCommand.scala | 25 +++++++--------
.../org/apache/spark/util/CleanFiles.scala | 32 ++++++++++++++------
.../apache/spark/util/CarbonCommandSuite.scala | 3 +-
4 files changed, 59 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b479617/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index d514f77..c02ba0a 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -98,34 +98,46 @@ object CarbonStore {
}
}
+ /**
+ * The method deletes all data if forceTableCLean <true> and lean garbage segment
+ * (MARKED_FOR_DELETE state) if forceTableCLean <false>
+ *
+ * @param dbName : Database name
+ * @param tableName : Table name
+ * @param tablePath : Table path
+ * @param carbonTable : CarbonTable Object <null> in case of force clean
+ * @param forceTableClean : <true> for force clean it will delete all data
+ * <false> it will clean garbage segment (MARKED_FOR_DELETE state)
+ * @param currentTablePartitions : Hive Partitions details
+ */
def cleanFiles(
dbName: String,
tableName: String,
- storePath: String,
+ tablePath: String,
carbonTable: CarbonTable,
forceTableClean: Boolean,
currentTablePartitions: Option[Seq[String]] = None): Unit = {
LOGGER.audit(s"The clean files request has been received for $dbName.$tableName")
var carbonCleanFilesLock: ICarbonLock = null
- var absoluteTableIdentifier: AbsoluteTableIdentifier = null
- if (forceTableClean) {
- absoluteTableIdentifier = AbsoluteTableIdentifier.from(storePath, dbName, tableName)
+ val absoluteTableIdentifier = if (forceTableClean) {
+ AbsoluteTableIdentifier.from(tablePath, dbName, tableName)
} else {
- absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
+ carbonTable.getAbsoluteTableIdentifier
}
try {
val errorMsg = "Clean files request is failed for " +
s"$dbName.$tableName" +
". Not able to acquire the clean files lock due to another clean files " +
"operation is running in the background."
- carbonCleanFilesLock =
- CarbonLockUtil.getLockObject(absoluteTableIdentifier, LockUsage.CLEAN_FILES_LOCK, errorMsg)
+ // in case of force clean the lock is not required
if (forceTableClean) {
- val absIdent = AbsoluteTableIdentifier.from(storePath, dbName, tableName)
FileFactory.deleteAllCarbonFilesOfDir(
- FileFactory.getCarbonFile(absIdent.getTablePath,
- FileFactory.getFileType(absIdent.getTablePath)))
+ FileFactory.getCarbonFile(absoluteTableIdentifier.getTablePath,
+ FileFactory.getFileType(absoluteTableIdentifier.getTablePath)))
} else {
+ carbonCleanFilesLock =
+ CarbonLockUtil
+ .getLockObject(absoluteTableIdentifier, LockUsage.CLEAN_FILES_LOCK, errorMsg)
DataLoadingUtil.deleteLoadsAndUpdateMetadata(
isForceDeletion = true, carbonTable)
CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, true)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b479617/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
index 303c3ef..4b68bd0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.events.{CleanFilesPostEvent, CleanFilesPreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.spark.util.CommonUtil
@@ -70,12 +70,13 @@ case class CarbonCleanFilesCommand(
databaseNameOp: Option[String], tableName: String): Unit = {
val dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
val databaseLocation = CarbonEnv.getDatabaseLocation(dbName, sparkSession)
+ val tablePath = databaseLocation + CarbonCommonConstants.FILE_SEPARATOR + tableName
CarbonStore.cleanFiles(
- dbName,
- tableName,
- databaseLocation,
- null,
- forceTableClean)
+ dbName = dbName,
+ tableName = tableName,
+ tablePath = tablePath,
+ carbonTable = null, // in case of delete all data carbonTable is not required.
+ forceTableClean = forceTableClean)
}
private def cleanGarbageData(sparkSession: SparkSession,
@@ -90,12 +91,12 @@ case class CarbonCleanFilesCommand(
None
}
CarbonStore.cleanFiles(
- CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession),
- tableName,
- CarbonProperties.getStorePath,
- carbonTable,
- forceTableClean,
- partitions)
+ dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession),
+ tableName = tableName,
+ tablePath = carbonTable.getTablePath,
+ carbonTable = carbonTable,
+ forceTableClean = forceTableClean,
+ currentTablePartitions = partitions)
}
// Clean garbage data in all tables in all databases
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b479617/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
index eba7dcd..d4d9a84 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
@@ -29,19 +29,30 @@ import org.apache.carbondata.api.CarbonStore
object CleanFiles {
/**
- * Clean the stale segments from table
- * @param spark
- * @param dbName
- * @param tableName
- * @param storePath
- * @param forceTableClean if true, it deletes the table and its contents with force.It does not
+ * The method deletes all data if forceTableCLean <true> and lean garbage segment
+ * (MARKED_FOR_DELETE state) if forceTableCLean <false>
+ *
+ * @param spark : Database name
+ * @param dbName : Table name
+ * @param tableName : Table path
+ * @param forceTableClean : if true, it deletes the table and its contents with force.It does not
* drop table from hive metastore so should be very careful to use it.
*/
def cleanFiles(spark: SparkSession, dbName: String, tableName: String,
- storePath: String, forceTableClean: Boolean = false): Unit = {
+ forceTableClean: Boolean = false): Unit = {
TableAPIUtil.validateTableExists(spark, dbName, tableName)
- val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(spark)
- CarbonStore.cleanFiles(dbName, tableName, storePath, carbonTable, forceTableClean)
+ val tablePath = CarbonEnv.getTablePath(Some(dbName), tableName)(spark)
+ val carbonTable = if (!forceTableClean) {
+ CarbonEnv.getCarbonTable(Some(dbName), tableName)(spark)
+ } else {
+ null
+ }
+ CarbonStore.cleanFiles(
+ dbName = dbName,
+ tableName = tableName,
+ tablePath = tablePath,
+ carbonTable = carbonTable,
+ forceTableClean = forceTableClean)
}
def main(args: Array[String]): Unit = {
@@ -60,6 +71,7 @@ object CleanFiles {
val spark = TableAPIUtil.spark(storePath, s"CleanFiles: $dbName.$tableName")
CarbonEnv.getInstance(spark).carbonMetastore.
checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
- cleanFiles(spark, dbName, tableName, storePath, forceTableClean)
+
+ cleanFiles(spark, dbName, tableName, forceTableClean)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b479617/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
index e493179..8ff6cab 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/util/CarbonCommandSuite.scala
@@ -142,7 +142,8 @@ class CarbonCommandSuite extends Spark2QueryTest with BeforeAndAfterAll {
dropTable(table)
createAndLoadTestTable(table, "csv_table")
CleanFiles.main(Array(s"${location}", table, "true"))
- val tablePath = s"${location}${File.separator}default${File.separator}$table"
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", table)
+ val tablePath = carbonTable.getTablePath
val f = new File(tablePath)
assert(!f.exists())
[36/50] [abbrv] carbondata git commit: [CARBONDATA-2095]Copy data of
GenericInternalRow for each iteration when converting stream segment to batch
segment
Posted by ja...@apache.org.
[CARBONDATA-2095]Copy data of GenericInternalRow for each iteration when converting stream segment to batch segment
This closes #1881
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2081fba7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2081fba7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2081fba7
Branch: refs/heads/carbonstore
Commit: 2081fba7f845c57fe4332bc8f33680a91c842720
Parents: 9ec64a6
Author: Zhang Zhichao <44...@qq.com>
Authored: Tue Jan 30 16:52:46 2018 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Tue Jan 30 18:39:40 2018 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/streaming/StreamHandoffRDD.scala | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2081fba7/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index d092580..a96ab32 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -75,11 +75,14 @@ class StreamingRawResultIterator(
}
override def next(): Array[Object] = {
- recordReader
+ val rowTmp = recordReader
.getCurrentValue
.asInstanceOf[GenericInternalRow]
.values
.asInstanceOf[Array[Object]]
+ val row = new Array[Object](rowTmp.length)
+ System.arraycopy(rowTmp, 0, row, 0, rowTmp.length)
+ row
}
}
[33/50] [abbrv] carbondata git commit: [CARBONDATA-1909] Load is
failing during insert into operation when load is concurrently done to source
table
Posted by ja...@apache.org.
[CARBONDATA-1909] Load is failing during insert into operation when load is concurrently done to source table
This closes #1693
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ab763474
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ab763474
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ab763474
Branch: refs/heads/carbonstore
Commit: ab763474f9a8191d84ea742a8b6ee615d310999a
Parents: a597c2f
Author: Manohar <ma...@gmail.com>
Authored: Wed Dec 20 15:09:45 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Mon Jan 29 15:30:53 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 18 +++++++++++
.../management/CarbonInsertIntoCommand.scala | 34 ++++++++++++++++++--
2 files changed, 49 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab763474/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 13c8a42..f46feef 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -865,6 +865,24 @@ public final class CarbonCommonConstants {
public static final String CARBON_MERGE_SORT_PREFETCH_DEFAULT = "true";
/**
+ * If we are executing insert into query from source table using select statement
+ * & loading the same source table concurrently, when select happens on source table
+ * during the data load , it gets new record for which dictionary is not generated,
+ * So there will be inconsistency. To avoid this condition we can persist the dataframe
+ * into MEMORY_AND_DISK and perform insert into operation. By default this value
+ * will be false because no need to persist the dataframe in all cases. If user want
+ * to run load and insert queries on source table concurrently then user can enable this flag
+ */
+ @CarbonProperty
+ public static final String CARBON_INSERT_PERSIST_ENABLED = "carbon.insert.persist.enable";
+
+ /**
+ * by default rdd will not be persisted in the insert case.
+
+ */
+ public static final String CARBON_INSERT_PERSIST_ENABLED_DEFAULT = "false";
+
+ /**
* default name of data base
*/
public static final String DATABASE_DEFAULT_NAME = "default";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ab763474/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
index 626cdba..86d6759 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
@@ -18,9 +18,14 @@
package org.apache.spark.sql.execution.command.management
import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Dataset, Row, SparkSession}
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.encoders.RowEncoder
+import org.apache.spark.sql.catalyst.plans.logical.{GlobalLimit, LogicalPlan}
import org.apache.spark.sql.execution.command.{AtomicRunnableCommand, DataCommand}
+import org.apache.spark.storage.StorageLevel
+import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.util.CarbonSparkUtil
case class CarbonInsertIntoCommand(
@@ -33,7 +38,26 @@ case class CarbonInsertIntoCommand(
var loadCommand: CarbonLoadDataCommand = _
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
- val df = Dataset.ofRows(sparkSession, child)
+ val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getName)
+ def containsLimit(plan: LogicalPlan): Boolean = {
+ plan find {
+ case limit: GlobalLimit => true
+ case other => false
+ } isDefined
+ }
+ val isPersistEnabledUserValue = CarbonProperties.getInstance
+ .getProperty(CarbonCommonConstants.CARBON_INSERT_PERSIST_ENABLED,
+ CarbonCommonConstants.CARBON_INSERT_PERSIST_ENABLED_DEFAULT)
+ val isPersistRequired =
+ isPersistEnabledUserValue.equalsIgnoreCase("true") || containsLimit(child)
+ val df =
+ if (isPersistRequired) {
+ LOGGER.audit("Persist enabled for Insert operation")
+ Dataset.ofRows(sparkSession, child)
+ .persist(StorageLevel.MEMORY_AND_DISK)
+ } else {
+ Dataset.ofRows(sparkSession, child)
+ }
val header = relation.tableSchema.get.fields.map(_.name).mkString(",")
loadCommand = CarbonLoadDataCommand(
databaseNameOp = Some(relation.carbonRelation.databaseName),
@@ -48,7 +72,11 @@ case class CarbonInsertIntoCommand(
tableInfoOp = None,
internalOptions = Map.empty,
partition = partition)
- loadCommand.processMetadata(sparkSession)
+ val load = loadCommand.processMetadata(sparkSession)
+ if (isPersistRequired) {
+ df.unpersist()
+ }
+ load
}
override def processData(sparkSession: SparkSession): Seq[Row] = {
if (null != loadCommand) {
[37/50] [abbrv] carbondata git commit: [CARBONDATA-1796] While
submiting new job, pass credentials in jobConf object
Posted by ja...@apache.org.
[CARBONDATA-1796] While submiting new job, pass credentials in jobConf object
This closes #1855
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b421c246
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b421c246
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b421c246
Branch: refs/heads/carbonstore
Commit: b421c24689b18429f13fd150ab4dd422c61ca622
Parents: 2081fba
Author: akashrn5 <ak...@gmail.com>
Authored: Wed Jan 24 14:26:28 2018 +0530
Committer: QiangCai <qi...@qq.com>
Committed: Tue Jan 30 18:52:25 2018 +0800
----------------------------------------------------------------------
.../spark/rdd/CarbonIUDMergerRDD.scala | 3 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 2 ++
.../carbondata/spark/rdd/CarbonScanRDD.scala | 7 +++-
.../scala/org/apache/spark/util/SparkUtil.scala | 35 ++------------------
.../spark/rdd/CarbonDataRDDFactory.scala | 5 +--
5 files changed, 15 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
index e8180cd..4378c15 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
@@ -25,9 +25,9 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.{Partition, SparkContext}
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.execution.command.CarbonMergerMapping
-import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
@@ -57,6 +57,7 @@ class CarbonIUDMergerRDD[K, V](
tablePath, new CarbonTableIdentifier(databaseName, factTableName, tableId)
)
val jobConf: JobConf = new JobConf(new Configuration)
+ SparkHadoopUtil.get.addCredentials(jobConf)
val job: Job = new Job(jobConf)
val format = CarbonInputFormatUtil.createCarbonInputFormat(absoluteTableIdentifier, job)
val defaultParallelism = sparkContext.defaultParallelism
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 8d7b044..c482a92 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.Job
import org.apache.spark._
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.execution.command.{CarbonMergerMapping, NodeInfo}
import org.apache.spark.sql.hive.DistributionUtil
import org.apache.spark.sql.util.CarbonException
@@ -276,6 +277,7 @@ class CarbonMergerRDD[K, V](
val updateStatusManager: SegmentUpdateStatusManager = new SegmentUpdateStatusManager(
absoluteTableIdentifier)
val jobConf: JobConf = new JobConf(new Configuration)
+ SparkHadoopUtil.get.addCredentials(jobConf)
val job: Job = new Job(jobConf)
val format = CarbonInputFormatUtil.createCarbonInputFormat(absoluteTableIdentifier, job)
CarbonTableInputFormat.setPartitionsToPrune(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index f2c3060..49c0225 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -26,9 +26,11 @@ import scala.collection.mutable.ArrayBuffer
import scala.util.Random
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce._
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
import org.apache.spark._
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.hive.DistributionUtil
import org.apache.spark.sql.SparkSession
@@ -81,7 +83,10 @@ class CarbonScanRDD(
@transient val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
override def getPartitions: Array[Partition] = {
- val job = Job.getInstance(new Configuration())
+ val conf = new Configuration()
+ val jobConf = new JobConf(conf)
+ SparkHadoopUtil.get.addCredentials(jobConf)
+ val job = Job.getInstance(jobConf)
val format = prepareInputFormatForDriver(job.getConfiguration)
// initialise query_id for job
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
index 9c37640..4635fc7 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
@@ -19,8 +19,10 @@ package org.apache.spark.util
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.{LongWritable, Text}
+import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
import org.apache.spark.{SparkContext, TaskContext}
+import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.rdd.{NewHadoopPartition, NewHadoopRDD}
import org.apache.carbondata.processing.loading.csvinput.BlockDetails
@@ -37,37 +39,4 @@ object SparkUtil {
}
}
- /**
- * get file splits,return Array[BlockDetails], if file path is empty,then return empty Array
- *
- */
- def getSplits(path: String, sc: SparkContext): Array[BlockDetails] = {
- val filePath = FileUtils.getPaths(path)
- if (filePath == null || filePath.isEmpty) {
- // return a empty block details
- Array[BlockDetails]()
- } else {
- // clone the hadoop configuration
- val hadoopConfiguration = new Configuration(sc.hadoopConfiguration)
- // set folder or file
- hadoopConfiguration.set(FileInputFormat.INPUT_DIR, filePath)
- hadoopConfiguration.set(FileInputFormat.INPUT_DIR_RECURSIVE, "true")
- val newHadoopRDD = new NewHadoopRDD[LongWritable, Text](
- sc,
- classOf[org.apache.hadoop.mapreduce.lib.input.TextInputFormat],
- classOf[LongWritable],
- classOf[Text],
- hadoopConfiguration)
- val splits: Array[FileSplit] = newHadoopRDD.getPartitions.map { part =>
- part.asInstanceOf[NewHadoopPartition].serializableHadoopSplit.value.asInstanceOf[FileSplit]
- }
- splits.map { block =>
- new BlockDetails(block.getPath,
- block.getStart,
- block.getLength,
- block.getLocations
- )
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b421c246/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 809c8ff..8212e85 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -1030,9 +1030,10 @@ object CarbonDataRDDFactory {
org.apache.hadoop.io.compress.BZip2Codec""".stripMargin)
CommonUtil.configSplitMaxSize(sqlContext.sparkContext, filePaths, hadoopConf)
-
+ val jobConf = new JobConf(hadoopConf)
+ SparkHadoopUtil.get.addCredentials(jobConf)
val inputFormat = new org.apache.hadoop.mapreduce.lib.input.TextInputFormat
- val jobContext = new Job(hadoopConf)
+ val jobContext = new Job(jobConf)
val rawSplits = inputFormat.getSplits(jobContext).toArray
val blockList = rawSplits.map { inputSplit =>
val fileSplit = inputSplit.asInstanceOf[FileSplit]
[24/50] [abbrv] carbondata git commit: [CARBONDATA-2066] Add Local
keyword when loading data to hive table
Posted by ja...@apache.org.
[CARBONDATA-2066] Add Local keyword when loading data to hive table
Add Local keyword when loading data to hive table
This closes #1845
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2e6bc66f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2e6bc66f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2e6bc66f
Branch: refs/heads/carbonstore
Commit: 2e6bc66f471c179780aa472efc30d51f5e25e743
Parents: f62f2ca
Author: Raghunandan S <ca...@gmail.com>
Authored: Tue Jan 23 12:59:34 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Sun Jan 28 14:33:04 2018 +0800
----------------------------------------------------------------------
.../spark/testsuite/primitiveTypes/DoubleDataTypeTestCase.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2e6bc66f/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/DoubleDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/DoubleDataTypeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/DoubleDataTypeTestCase.scala
index 1d63eda..fe80cd3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/DoubleDataTypeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/DoubleDataTypeTestCase.scala
@@ -82,7 +82,7 @@ class DoubleDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
sql("create table uniq_carbon(name string, double_column double) stored by 'carbondata' TBLPROPERTIES ('DICTIONARY_INCLUDE'='double_column')")
sql(s"load data inpath '$resourcesPath/uniq.csv' into table uniq_carbon")
sql("create table uniq_hive(name string, double_column double) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
- sql(s"load data inpath '$resourcesPath/uniqwithoutheader.csv' into table uniq_hive")
+ sql(s"load data local inpath '$resourcesPath/uniqwithoutheader.csv' into table uniq_hive")
checkAnswer(sql("select * from uniq_carbon where double_column>=11"),
sql("select * from uniq_hive where double_column>=11"))
}
[48/50] [abbrv] carbondata git commit: [CARBONDATA-1992] Remove
partitionId in CarbonTablePath
Posted by ja...@apache.org.
[CARBONDATA-1992] Remove partitionId in CarbonTablePath
In CarbonTablePath, there is a deprecated partition id which is always 0, it should be removed to avoid confusion.
This closes #1765
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3b4c51c0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3b4c51c0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3b4c51c0
Branch: refs/heads/carbonstore
Commit: 3b4c51c03a86443643f155442b41f73a78d07ce1
Parents: 8314ea2
Author: Jacky Li <ja...@qq.com>
Authored: Sat Jan 6 20:28:44 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 13:17:14 2018 +0800
----------------------------------------------------------------------
.../core/metadata/PartitionMapFileStore.java | 2 +-
.../core/mutate/CarbonUpdateUtil.java | 8 +-
.../core/statusmanager/LoadMetadataDetails.java | 2 +
.../SegmentUpdateStatusManager.java | 8 +-
.../apache/carbondata/core/util/CarbonUtil.java | 6 +-
.../core/util/path/CarbonTablePath.java | 55 +++---
.../CarbonFormatDirectoryStructureTest.java | 4 +-
.../hadoop/api/CarbonTableInputFormat.java | 2 +-
.../streaming/CarbonStreamRecordWriter.java | 2 +-
.../hadoop/test/util/StoreCreator.java | 1 -
.../presto/util/CarbonDataStoreCreator.scala | 1 -
.../dataload/TestLoadDataGeneral.scala | 2 +-
.../InsertIntoCarbonTableTestCase.scala | 4 +-
.../dataload/TestBatchSortDataLoad.scala | 3 +-
.../dataload/TestDataLoadWithFileName.scala | 2 +-
.../dataload/TestGlobalSortDataLoad.scala | 4 +-
.../testsuite/datamap/TestDataMapCommand.scala | 34 ++--
.../TestDataLoadingForPartitionTable.scala | 3 +-
.../StandardPartitionTableCleanTestCase.scala | 2 +-
...andardPartitionTableCompactionTestCase.scala | 2 +-
.../StandardPartitionTableLoadingTestCase.scala | 4 +-
.../load/DataLoadProcessBuilderOnSpark.scala | 1 -
.../load/DataLoadProcessorStepOnSpark.scala | 2 +-
.../spark/rdd/AlterTableLoadPartitionRDD.scala | 169 +++++++++----------
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 11 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala | 25 ++-
.../org/apache/spark/util/PartitionUtils.scala | 5 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 5 +-
.../datasources/CarbonFileFormat.scala | 1 -
.../partition/TestAlterPartitionTable.scala | 2 +-
.../bucketing/TableBucketingTestCase.scala | 2 +
.../loading/CarbonDataLoadConfiguration.java | 10 --
.../loading/DataLoadProcessBuilder.java | 1 -
.../loading/TableProcessingOperations.java | 3 +-
.../loading/model/CarbonLoadModel.java | 72 +-------
.../sort/impl/ParallelReadMergeSorterImpl.java | 4 +-
...arallelReadMergeSorterWithBucketingImpl.java | 15 +-
.../UnsafeBatchParallelReadMergeSorterImpl.java | 7 +-
...arallelReadMergeSorterWithBucketingImpl.java | 21 +--
.../CarbonRowDataWriterProcessorStepImpl.java | 33 ++--
.../steps/DataWriterBatchProcessorStepImpl.java | 25 +--
.../steps/DataWriterProcessorStepImpl.java | 22 +--
.../processing/merger/CarbonDataMergerUtil.java | 6 +-
.../merger/CompactionResultSortProcessor.java | 4 +-
.../sort/sortdata/SortParameters.java | 16 +-
.../store/CarbonFactDataHandlerModel.java | 7 +-
.../util/CarbonDataProcessorUtil.java | 12 +-
.../processing/util/CarbonLoaderUtil.java | 12 +-
.../processing/util/DeleteLoadFolders.java | 7 +-
.../carbondata/processing/StoreCreator.java | 1 -
.../carbondata/streaming/StreamHandoffRDD.scala | 1 -
.../streaming/StreamSinkFactory.scala | 2 +-
.../streaming/CarbonAppendableStreamSink.scala | 8 +-
53 files changed, 278 insertions(+), 385 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
index 355d083..a09feab 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
@@ -358,7 +358,7 @@ public class PartitionMapFileStore {
List<String> toBeDeletedIndexFiles = new ArrayList<>();
List<String> toBeDeletedDataFiles = new ArrayList<>();
// take the list of files from this segment.
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment.getLoadName());
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segment.getLoadName());
String partitionFilePath = getPartitionFilePath(segmentPath);
if (partitionFilePath != null) {
PartitionMapper partitionMapper = readPartitionMap(partitionFilePath);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index f4566ac..bace48e 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -297,9 +297,7 @@ public class CarbonUpdateUtil {
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
- // as of now considering only partition 0.
- String partitionId = "0";
- String partitionDir = carbonTablePath.getPartitionDir(partitionId);
+ String partitionDir = carbonTablePath.getPartitionDir();
CarbonFile file =
FileFactory.getCarbonFile(partitionDir, FileFactory.getFileType(partitionDir));
if (!file.exists()) {
@@ -380,7 +378,7 @@ public class CarbonUpdateUtil {
}
public static long getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
- String segmentDirPath = tablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentDirPath = tablePath.getCarbonDataDirectoryPath(segmentId);
// scan all the carbondata files and get the latest task ID.
CarbonFile segment =
@@ -441,7 +439,7 @@ public class CarbonUpdateUtil {
|| segment.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
// take the list of files from this segment.
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment.getLoadName());
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segment.getLoadName());
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index 85602bc..73a665d 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -123,10 +123,12 @@ public class LoadMetadataDetails implements Serializable {
*/
private FileFormat fileFormat = FileFormat.COLUMNAR_V3;
+ @Deprecated
public String getPartitionCount() {
return partitionCount;
}
+ @Deprecated
public void setPartitionCount(String partitionCount) {
this.partitionCount = partitionCount;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index df7eedd..d28b39f 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -197,7 +197,7 @@ public class SegmentUpdateStatusManager {
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
String endTimeStamp = "";
String startTimeStamp = "";
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
for (LoadMetadataDetails eachSeg : segmentDetails) {
@@ -292,7 +292,7 @@ public class SegmentUpdateStatusManager {
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
String segment = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID);
- String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment);
+ String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segment);
String completeBlockName = CarbonTablePath.addDataPartPrefix(
CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCK_ID)
+ CarbonCommonConstants.FACT_FILE_EXT);
@@ -424,7 +424,7 @@ public class SegmentUpdateStatusManager {
.getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
@@ -862,7 +862,7 @@ public class SegmentUpdateStatusManager {
// filter out the fact files.
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 5d7a09f..cd0f4b8 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1082,7 +1082,7 @@ public final class CarbonUtil {
// geting the index file path
//TODO need to pass proper partition number when partiton will be supported
String carbonIndexFilePath = carbonTablePath
- .getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
+ .getCarbonIndexFilePath(taskId, tableBlockInfoList.get(0).getSegmentId(),
bucketNumber, CarbonTablePath.DataFileUtil
.getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
tableBlockInfoList.get(0).getVersion());
@@ -1325,7 +1325,7 @@ public final class CarbonUtil {
// geting the index file path
//TODO need to pass proper partition number when partiton will be supported
String carbonIndexFilePath = carbonTablePath
- .getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
+ .getCarbonIndexFilePath(taskId, tableBlockInfoList.get(0).getSegmentId(),
bucketNumber, CarbonTablePath.DataFileUtil
.getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
tableBlockInfoList.get(0).getVersion());
@@ -2268,7 +2268,7 @@ public final class CarbonUtil {
long carbonDataSize = 0L;
long carbonIndexSize = 0L;
HashMap<String, Long> dataAndIndexSize = new HashMap<String, Long>();
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
+ String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
FileFactory.FileType fileType = FileFactory.getFileType(segmentPath);
switch (fileType) {
case HDFS:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 9e66657..36eaa56 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -255,15 +255,14 @@ public class CarbonTablePath extends Path {
/**
* Gets absolute path of data file
*
- * @param partitionId unique partition identifier
* @param segmentId unique partition identifier
* @param filePartNo data file part number
* @param factUpdateTimeStamp unique identifier to identify an update
* @return absolute path of data file stored in carbon data format
*/
- public String getCarbonDataFilePath(String partitionId, String segmentId, Integer filePartNo,
- Long taskNo, int batchNo, int bucketNumber, String factUpdateTimeStamp) {
- return getSegmentDir(partitionId, segmentId) + File.separator + getCarbonDataFileName(
+ public String getCarbonDataFilePath(String segmentId, Integer filePartNo, Long taskNo,
+ int batchNo, int bucketNumber, String factUpdateTimeStamp) {
+ return getSegmentDir(segmentId) + File.separator + getCarbonDataFileName(
filePartNo, taskNo, bucketNumber, batchNo, factUpdateTimeStamp);
}
@@ -272,13 +271,12 @@ public class CarbonTablePath extends Path {
* based on task id
*
* @param taskId task id of the file
- * @param partitionId partition number
* @param segmentId segment number
* @return full qualified carbon index path
*/
- public String getCarbonIndexFilePath(final String taskId, final String partitionId,
- final String segmentId, final String bucketNumber) {
- String segmentDir = getSegmentDir(partitionId, segmentId);
+ public String getCarbonIndexFilePath(final String taskId, final String segmentId,
+ final String bucketNumber) {
+ String segmentDir = getSegmentDir(segmentId);
CarbonFile carbonFile =
FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
@@ -294,9 +292,8 @@ public class CarbonTablePath extends Path {
if (files.length > 0) {
return files[0].getAbsolutePath();
} else {
- throw new RuntimeException("Missing Carbon index file for partition["
- + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId
- + "]");
+ throw new RuntimeException("Missing Carbon index file for Segment[" + segmentId + "], "
+ + "taskId[" + taskId + "]");
}
}
@@ -304,8 +301,6 @@ public class CarbonTablePath extends Path {
* Below method will be used to get the carbon index file path
* @param taskId
* task id
- * @param partitionId
- * partition id
* @param segmentId
* segment id
* @param bucketNumber
@@ -314,28 +309,27 @@ public class CarbonTablePath extends Path {
* timestamp
* @return carbon index file path
*/
- public String getCarbonIndexFilePath(String taskId, String partitionId, String segmentId,
- String bucketNumber, String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
+ public String getCarbonIndexFilePath(String taskId, String segmentId, String bucketNumber,
+ String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
switch (columnarFormatVersion) {
case V1:
case V2:
- return getCarbonIndexFilePath(taskId, partitionId, segmentId, bucketNumber);
+ return getCarbonIndexFilePath(taskId, segmentId, bucketNumber);
default:
- String segmentDir = getSegmentDir(partitionId, segmentId);
+ String segmentDir = getSegmentDir(segmentId);
return segmentDir + File.separator + getCarbonIndexFileName(taskId,
Integer.parseInt(bucketNumber), timeStamp);
}
}
- public String getCarbonIndexFilePath(String taskId, String partitionId, String segmentId,
- int batchNo, String bucketNumber, String timeStamp,
- ColumnarFormatVersion columnarFormatVersion) {
+ public String getCarbonIndexFilePath(String taskId, String segmentId, int batchNo,
+ String bucketNumber, String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
switch (columnarFormatVersion) {
case V1:
case V2:
- return getCarbonIndexFilePath(taskId, partitionId, segmentId, bucketNumber);
+ return getCarbonIndexFilePath(taskId, segmentId, bucketNumber);
default:
- String segmentDir = getSegmentDir(partitionId, segmentId);
+ String segmentDir = getSegmentDir(segmentId);
return segmentDir + File.separator + getCarbonIndexFileName(Long.parseLong(taskId),
Integer.parseInt(bucketNumber), batchNo, timeStamp);
}
@@ -349,12 +343,11 @@ public class CarbonTablePath extends Path {
/**
* Gets absolute path of data file
*
- * @param partitionId unique partition identifier
* @param segmentId unique partition identifier
* @return absolute path of data file stored in carbon data format
*/
- public String getCarbonDataDirectoryPath(String partitionId, String segmentId) {
- return getSegmentDir(partitionId, segmentId);
+ public String getCarbonDataDirectoryPath(String segmentId) {
+ return getSegmentDir(segmentId);
}
/**
@@ -392,12 +385,16 @@ public class CarbonTablePath extends Path {
return segmentDir + File.separator + getCarbonStreamIndexFileName();
}
- public String getSegmentDir(String partitionId, String segmentId) {
- return getPartitionDir(partitionId) + File.separator + SEGMENT_PREFIX + segmentId;
+ public String getSegmentDir(String segmentId) {
+ return getPartitionDir() + File.separator + SEGMENT_PREFIX + segmentId;
}
- public String getPartitionDir(String partitionId) {
- return getFactDir() + File.separator + PARTITION_PREFIX + partitionId;
+ // This partition is not used in any code logic, just keep backward compatibility
+ public static final String DEPRECATED_PATITION_ID = "0";
+
+ public String getPartitionDir() {
+ return getFactDir() + File.separator + PARTITION_PREFIX +
+ CarbonTablePath.DEPRECATED_PATITION_ID;
}
private String getMetaDataDir() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
index 5549806..a1ccab3 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/path/CarbonFormatDirectoryStructureTest.java
@@ -53,8 +53,8 @@ public class CarbonFormatDirectoryStructureTest {
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.dictmeta"));
assertTrue(carbonTablePath.getSortIndexFilePath("t1_c1").replace("\\", "/")
.equals(CARBON_STORE + "/d1/t1/Metadata/t1_c1.sortindex"));
- assertTrue(carbonTablePath.getCarbonDataFilePath("1", "2", 3, 4L, 0, 0, "999").replace("\\", "/")
- .equals(CARBON_STORE + "/d1/t1/Fact/Part1/Segment_2/part-3-4_batchno0-0-999.carbondata"));
+ assertTrue(carbonTablePath.getCarbonDataFilePath("2", 3, 4L, 0, 0, "999").replace("\\", "/")
+ .equals(CARBON_STORE + "/d1/t1/Fact/Part0/Segment_2/part-3-4_batchno0-0-999.carbondata"));
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index a1887f0..6f1e123 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -480,7 +480,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
for (String segmentId : streamSegments) {
- String segmentDir = tablePath.getSegmentDir("0", segmentId);
+ String segmentDir = tablePath.getSegmentDir(segmentId);
FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
if (FileFactory.isFileExist(segmentDir, fileType)) {
String indexName = CarbonTablePath.getCarbonStreamIndexFileName();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
index 364a6a6..3ef8afc 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
@@ -129,7 +129,7 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
CarbonTablePath tablePath =
CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier());
- segmentDir = tablePath.getSegmentDir("0", segmentId);
+ segmentDir = tablePath.getSegmentDir(segmentId);
fileName = CarbonTablePath.getCarbonDataFileName(0, taskNo, 0, 0, "0");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index fbf33d6..ac17c4e 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -154,7 +154,6 @@ public class StoreCreator {
loadModel.setCsvHeaderColumns(loadModel.getCsvHeader().split(","));
loadModel.setTaskNo("0");
loadModel.setSegmentId("0");
- loadModel.setPartitionId("0");
loadModel.setFactTimeStamp(System.currentTimeMillis());
loadModel.setMaxColumns("10");
return loadModel;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index 7b5c311..a41e738 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -135,7 +135,6 @@ object CarbonDataStoreCreator {
loadModel.setCsvHeaderColumns(loadModel.getCsvHeader.split(","))
loadModel.setTaskNo("0")
loadModel.setSegmentId("0")
- loadModel.setPartitionId("0")
loadModel.setFactTimeStamp(System.currentTimeMillis())
loadModel.setMaxColumns("15")
executeGraph(loadModel, storePath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index 09ca9e5..c84ae6b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -49,7 +49,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
tableName: String): Boolean = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(datbaseName, tableName)
val partitionPath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir("0")
+ .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir
val fileType: FileFactory.FileType = FileFactory.getFileType(partitionPath)
val carbonFile = FileFactory.getCarbonFile(partitionPath, fileType)
val segments: ArrayBuffer[String] = ArrayBuffer()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index d59f0b5..5cc4156 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -232,7 +232,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select count(*) from CarbonOverwrite"), sql("select count(*) from HiveOverwrite"))
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbonoverwrite")
val partitionPath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir("0")
+ .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir
val folder = new File(partitionPath)
assert(folder.isDirectory)
assert(folder.list().length == 1)
@@ -255,7 +255,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select count(*) from TCarbonSourceOverwrite"), sql("select count(*) from HiveOverwrite"))
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "tcarbonsourceoverwrite")
val partitionPath = CarbonStorePath
- .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir("0")
+ .getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier).getPartitionDir
val folder = new File(partitionPath)
assert(folder.isDirectory)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
index 4af9d54..42ac4df 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
@@ -193,9 +193,8 @@ class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
CarbonCommonConstants.DATABASE_DEFAULT_NAME,
tableName
)
- val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getCarbonDataDirectoryPath("0", segmentNo)
+ val segmentDir = carbonTablePath.getCarbonDataDirectoryPath(segmentNo)
new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
index dae0962..db0a62c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithFileName.scala
@@ -49,7 +49,7 @@ class TestDataLoadWithFileName extends QueryTest with BeforeAndAfterAll {
val indexReader = new CarbonIndexFileReader()
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "test_table_v3")
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getCarbonDataDirectoryPath("0", "0")
+ val segmentDir = carbonTablePath.getCarbonDataDirectoryPath("0")
val carbonIndexPaths = new File(segmentDir)
.listFiles(new FilenameFilter {
override def accept(dir: File, name: String): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index 9ce9675..8bd814a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -271,7 +271,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql(s"LOAD DATA LOCAL INPATH '$inputPath' INTO TABLE carbon_globalsort")
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", "carbon_globalsort")
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getSegmentDir("0", "0")
+ val segmentDir = carbonTablePath.getSegmentDir("0")
assertResult(5)(new File(segmentDir).listFiles().length)
}
@@ -356,7 +356,7 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
private def getIndexFileCount(tableName: String, segmentNo: String = "0"): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", tableName)
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- val segmentDir = carbonTablePath.getCarbonDataDirectoryPath("0", segmentNo)
+ val segmentDir = carbonTablePath.getCarbonDataDirectoryPath(segmentNo)
new SegmentIndexFileStore().getIndexFilesFromSegment(segmentDir).size()
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index 0860da1..f0a5277 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -217,19 +217,27 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
}
test("test if preaggregate load is successfull for hivemetastore") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
- .stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
+ sql("DROP TABLE IF EXISTS maintable")
+ sql(
+ """
+ | CREATE TABLE maintable(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(
+ s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
+
+ .stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
+ checkAnswer(sql(s"select * from maintable_preagg_sum"),
+ Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
+ } finally {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ }
}
test("test preaggregate load for decimal column for hivemetastore") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
index ed151bd..0a21aed 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala
@@ -63,7 +63,7 @@ class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
@@ -87,6 +87,7 @@ class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll
| projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,
| utilization int,salary int)
| PARTITIONED BY (empno int)
+ |
| STORED BY 'org.apache.carbondata.format'
| TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3')
""".stripMargin)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
index 2b0dd09..5427981 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
@@ -53,7 +53,7 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
index 22ebd80..f4b6e0e 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
@@ -53,7 +53,7 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 31d2598..7fcfb29 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -68,7 +68,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
@@ -335,7 +335,7 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default_mergeindexpartitionthree")
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", "0")
+ val segmentDir = tablePath.getCarbonDataDirectoryPath("0")
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val files = carbonFile.listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = CarbonTablePath.isCarbonIndexFile(file.getName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
index 781b484..455f445 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
@@ -72,7 +72,6 @@ object DataLoadProcessBuilderOnSpark {
.map(DataLoadProcessorStepOnSpark.toStringArrayRow(_, columnCount))
}
- model.setPartitionId("0")
val sc = sparkSession.sparkContext
val modelBroadcast = sc.broadcast(model)
val partialSuccessAccum = sc.accumulator(0, "Partial Success Accumulator")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 21de003..834c1a6 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -194,7 +194,7 @@ object DataLoadProcessorStepOnSpark {
dataWriter = new DataWriterProcessorStepImpl(conf)
- val dataHandlerModel = dataWriter.getDataHandlerModel(0)
+ val dataHandlerModel = dataWriter.getDataHandlerModel
var dataHandler: CarbonFactHandler = null
var rowsNotExist = true
while (rows.hasNext) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
index 35a8ea7..c01ebed 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
@@ -41,102 +41,95 @@ class AlterTableLoadPartitionRDD[K, V](alterPartitionModel: AlterPartitionModel,
identifier: AbsoluteTableIdentifier,
prev: RDD[Array[AnyRef]]) extends RDD[(K, V)](prev) {
- var storeLocation: String = null
- val carbonLoadModel = alterPartitionModel.carbonLoadModel
- val segmentId = alterPartitionModel.segmentId
- val oldPartitionIds = alterPartitionModel.oldPartitionIds
- val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val databaseName = carbonTable.getDatabaseName
- val factTableName = carbonTable.getTableName
- val partitionInfo = carbonTable.getPartitionInfo(factTableName)
+ var storeLocation: String = null
+ val carbonLoadModel = alterPartitionModel.carbonLoadModel
+ val segmentId = alterPartitionModel.segmentId
+ val oldPartitionIds = alterPartitionModel.oldPartitionIds
+ val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ val databaseName = carbonTable.getDatabaseName
+ val factTableName = carbonTable.getTableName
+ val partitionInfo = carbonTable.getPartitionInfo(factTableName)
- override protected def getPartitions: Array[Partition] = {
- val sc = alterPartitionModel.sqlContext.sparkContext
- sc.setLocalProperty("spark.scheduler.pool", "DDL")
- sc.setLocalProperty("spark.job.interruptOnCancel", "true")
- firstParent[Array[AnyRef]].partitions
- }
+ override protected def getPartitions: Array[Partition] = {
+ val sc = alterPartitionModel.sqlContext.sparkContext
+ sc.setLocalProperty("spark.scheduler.pool", "DDL")
+ sc.setLocalProperty("spark.job.interruptOnCancel", "true")
+ firstParent[Array[AnyRef]].partitions
+ }
- override def compute(split: Partition, context: TaskContext): Iterator[(K, V)] = {
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
- val rows = firstParent[Array[AnyRef]].iterator(split, context).toList.asJava
- val iter = new Iterator[(K, V)] {
- val partitionId = partitionInfo.getPartitionId(split.index)
- carbonLoadModel.setTaskNo(String.valueOf(partitionId))
- carbonLoadModel.setSegmentId(segmentId)
- carbonLoadModel.setPartitionId("0")
- val tempLocationKey = CarbonDataProcessorUtil
- .getTempStoreLocationKey(carbonLoadModel.getDatabaseName,
- carbonLoadModel.getTableName,
- segmentId,
- carbonLoadModel.getTaskNo,
- false,
- true)
- // this property is used to determine whether temp location for carbon is inside
- // container temp dir or is yarn application directory.
- val carbonUseLocalDir = CarbonProperties.getInstance()
- .getProperty("carbon.use.local.dir", "false")
+ override def compute(split: Partition, context: TaskContext): Iterator[(K, V)] = {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+ val rows = firstParent[Array[AnyRef]].iterator(split, context).toList.asJava
+ val iter = new Iterator[(K, V)] {
+ val partitionId = partitionInfo.getPartitionId(split.index)
+ carbonLoadModel.setTaskNo(String.valueOf(partitionId))
+ carbonLoadModel.setSegmentId(segmentId)
+ val tempLocationKey = CarbonDataProcessorUtil
+ .getTempStoreLocationKey(carbonLoadModel.getDatabaseName,
+ carbonLoadModel.getTableName,
+ segmentId,
+ carbonLoadModel.getTaskNo,
+ false,
+ true)
+ // this property is used to determine whether temp location for carbon is inside
+ // container temp dir or is yarn application directory.
+ val carbonUseLocalDir = CarbonProperties.getInstance()
+ .getProperty("carbon.use.local.dir", "false")
- if (carbonUseLocalDir.equalsIgnoreCase("true")) {
+ if (carbonUseLocalDir.equalsIgnoreCase("true")) {
- val storeLocations = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
- if (null != storeLocations && storeLocations.nonEmpty) {
- storeLocation = storeLocations(Random.nextInt(storeLocations.length))
- }
- if (storeLocation == null) {
- storeLocation = System.getProperty("java.io.tmpdir")
- }
- } else {
- storeLocation = System.getProperty("java.io.tmpdir")
- }
- storeLocation = storeLocation + '/' + System.nanoTime() + '/' + split.index
- CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation)
- LOGGER.info(s"Temp storeLocation taken is $storeLocation")
+ val storeLocations = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
+ if (null != storeLocations && storeLocations.nonEmpty) {
+ storeLocation = storeLocations(Random.nextInt(storeLocations.length))
+ }
+ if (storeLocation == null) {
+ storeLocation = System.getProperty("java.io.tmpdir")
+ }
+ } else {
+ storeLocation = System.getProperty("java.io.tmpdir")
+ }
+ storeLocation = storeLocation + '/' + System.nanoTime() + '/' + split.index
+ CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation)
+ LOGGER.info(s"Temp storeLocation taken is $storeLocation")
- val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(databaseName,
- factTableName,
- carbonLoadModel.getTaskNo,
- "0",
- segmentId,
- false,
- true
- )
+ val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ databaseName, factTableName, carbonLoadModel.getTaskNo, segmentId, false, true)
- val loadStatus = if (rows.isEmpty) {
- LOGGER.info("After repartition this split, NO target rows to write back.")
- true
- } else {
- val segmentProperties = PartitionUtils.getSegmentProperties(identifier,
- segmentId, partitionIds.toList, oldPartitionIds, partitionInfo, carbonTable)
- val processor = new RowResultProcessor(
- carbonTable,
- carbonLoadModel,
- segmentProperties,
- tempStoreLoc,
- bucketId)
- try {
- processor.execute(rows)
- } catch {
- case e: Exception =>
- sys.error(s"Exception when executing Row result processor ${e.getMessage}")
- } finally {
- TableProcessingOperations
- .deleteLocalDataLoadFolderLocation(carbonLoadModel, false, true)
- }
- }
+ val loadStatus = if (rows.isEmpty) {
+ LOGGER.info("After repartition this split, NO target rows to write back.")
+ true
+ } else {
+ val segmentProperties = PartitionUtils.getSegmentProperties(identifier,
+ segmentId, partitionIds.toList, oldPartitionIds, partitionInfo, carbonTable)
+ val processor = new RowResultProcessor(
+ carbonTable,
+ carbonLoadModel,
+ segmentProperties,
+ tempStoreLoc,
+ bucketId)
+ try {
+ processor.execute(rows)
+ } catch {
+ case e: Exception =>
+ sys.error(s"Exception when executing Row result processor ${ e.getMessage }")
+ } finally {
+ TableProcessingOperations
+ .deleteLocalDataLoadFolderLocation(carbonLoadModel, false, true)
+ }
+ }
- val loadResult = segmentId
- var finished = false
+ val loadResult = segmentId
+ var finished = false
- override def hasNext: Boolean = {
- !finished
- }
+ override def hasNext: Boolean = {
+ !finished
+ }
- override def next(): (K, V) = {
- finished = true
- result.getKey(loadResult, loadStatus)
- }
- }
- iter
+ override def next(): (K, V) = {
+ finished = true
+ result.getKey(loadResult, loadStatus)
+ }
}
+ iter
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index c482a92..a9f8667 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -203,16 +203,9 @@ class CarbonMergerRDD[K, V](
}
}
- val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(databaseName,
- factTableName,
- carbonLoadModel.getTaskNo,
- "0",
- mergeNumber,
- true,
- false
- )
+ val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ databaseName, factTableName, carbonLoadModel.getTaskNo, mergeNumber, true, false)
- carbonLoadModel.setPartitionId("0")
var processor: AbstractResultProcessor = null
if (restructuredBlockExists) {
processor = new CompactionResultSortProcessor(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 72d0484..1fa1689 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.compression.CompressorFactory
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, ThreadLocalTaskInfo}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.loading.{DataLoadExecutor, FailureCauses, TableProcessingOperations}
import org.apache.carbondata.processing.loading.csvinput.{BlockDetails, CSVInputFormat, CSVRecordReaderIterator}
import org.apache.carbondata.processing.loading.exception.NoRetryException
@@ -129,7 +130,8 @@ class SparkPartitionLoader(model: CarbonLoadModel,
System.setProperty("carbon.properties.filepath",
System.getProperty("user.dir") + '/' + "conf" + '/' + "carbon.properties")
}
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance.initPartitonInfo(model.getPartitionId)
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance.initPartitonInfo(
+ CarbonTablePath.DEPRECATED_PATITION_ID)
CarbonProperties.getInstance().addProperty("carbon.is.columnar.storage", "true")
CarbonProperties.getInstance().addProperty("carbon.dimension.split.value.in.columnar", "1")
CarbonProperties.getInstance().addProperty("carbon.is.fullyfilled.bits", "true")
@@ -219,14 +221,13 @@ class NewCarbonDataLoadRDD[K, V](
override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
- var partitionID = "0"
val loadMetadataDetails = new LoadMetadataDetails()
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
var model: CarbonLoadModel = _
val uniqueLoadStatusId =
carbonLoadModel.getTableName + CarbonCommonConstants.UNDERSCORE + theSplit.index
try {
- loadMetadataDetails.setPartitionCount(partitionID)
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID)
loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS)
val preFetch = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
@@ -264,7 +265,7 @@ class NewCarbonDataLoadRDD[K, V](
// So print the data load statistics only in case of non failure case
if (SegmentStatus.LOAD_FAILURE != loadMetadataDetails.getSegmentStatus) {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance
- .printStatisticsInfo(model.getPartitionId)
+ .printStatisticsInfo(CarbonTablePath.DEPRECATED_PATITION_ID)
}
}
@@ -287,8 +288,8 @@ class NewCarbonDataLoadRDD[K, V](
val fileList: java.util.List[String] = new java.util.ArrayList[String](
CarbonCommonConstants.CONSTANT_SIZE_TEN)
CarbonQueryUtil.splitFilePath(carbonLoadModel.getFactFilePath, fileList, ",")
- model = carbonLoadModel.getCopyWithPartition(partitionID, fileList,
- carbonLoadModel.getCsvHeader, carbonLoadModel.getCsvDelimiter)
+ model = carbonLoadModel.getCopyWithPartition(
+ carbonLoadModel.getCsvHeader, carbonLoadModel.getCsvDelimiter)
StandardLogService.setThreadName(StandardLogService
.getPartitionID(model.getCarbonDataLoadSchema.getCarbonTable.getTableUniqueName)
, ThreadLocalTaskInfo.getCarbonTaskInfo.getTaskId + "")
@@ -351,7 +352,6 @@ class NewDataFrameLoaderRDD[K, V](
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
- val partitionID = "0"
val loadMetadataDetails = new LoadMetadataDetails()
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
val model: CarbonLoadModel = carbonLoadModel
@@ -359,9 +359,8 @@ class NewDataFrameLoaderRDD[K, V](
carbonLoadModel.getTableName + CarbonCommonConstants.UNDERSCORE + theSplit.index
try {
- loadMetadataDetails.setPartitionCount(partitionID)
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID)
loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS)
- carbonLoadModel.setPartitionId(partitionID)
carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
carbonLoadModel.setPreFetch(false)
@@ -406,7 +405,7 @@ class NewDataFrameLoaderRDD[K, V](
// So print the data load statistics only in case of non failure case
if (SegmentStatus.LOAD_FAILURE != loadMetadataDetails.getSegmentStatus) {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance
- .printStatisticsInfo(model.getPartitionId)
+ .printStatisticsInfo(CarbonTablePath.DEPRECATED_PATITION_ID)
}
}
var finished = false
@@ -542,7 +541,6 @@ class PartitionTableDataLoaderRDD[K, V](
override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
- val partitionID = "0"
val loadMetadataDetails = new LoadMetadataDetails()
val executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
val model: CarbonLoadModel = carbonLoadModel
@@ -552,9 +550,8 @@ class PartitionTableDataLoaderRDD[K, V](
carbonLoadModel.getTableName + CarbonCommonConstants.UNDERSCORE + theSplit.index
try {
- loadMetadataDetails.setPartitionCount(partitionID)
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID)
loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS)
- carbonLoadModel.setPartitionId(partitionID)
carbonLoadModel.setTaskNo(String.valueOf(partitionInfo.getPartitionId(theSplit.index)))
carbonLoadModel.setPreFetch(false)
val recordReaders = Array[CarbonIterator[Array[AnyRef]]] {
@@ -590,7 +587,7 @@ class PartitionTableDataLoaderRDD[K, V](
// So print the data load statistics only in case of non failure case
if (SegmentStatus.LOAD_FAILURE != loadMetadataDetails.getSegmentStatus) {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance
- .printStatisticsInfo(model.getPartitionId)
+ .printStatisticsInfo(CarbonTablePath.DEPRECATED_PATITION_ID)
}
}
var finished = false
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
index 0498b25..3c871db 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/PartitionUtils.scala
@@ -190,8 +190,9 @@ object PartitionUtils {
val batchNo = CarbonTablePath.DataFileUtil.getBatchNoFromTaskNo(taskNo)
val taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskNo)
val bucketNumber = CarbonTablePath.DataFileUtil.getBucketNo(path)
- val indexFilePath = carbonTablePath.getCarbonIndexFilePath(String.valueOf(taskId), "0",
- segmentId, batchNo, String.valueOf(bucketNumber), timestamp, version)
+ val indexFilePath = carbonTablePath.getCarbonIndexFilePath(
+ String.valueOf(taskId), segmentId, batchNo, String.valueOf(bucketNumber),
+ timestamp, version)
// indexFilePath could be duplicated when multiple data file related to one index file
if (indexFilePath != null && !pathList.contains(indexFilePath)) {
pathList.add(indexFilePath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 8212e85..6f056a6 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -613,7 +613,6 @@ object CarbonDataRDDFactory {
override def getPartition(key: Any): Int = {
val segId = key.asInstanceOf[String]
- // partitionId
segmentIdIndex(segId) * parallelism + Random.nextInt(parallelism)
}
}
@@ -647,7 +646,6 @@ object CarbonDataRDDFactory {
val rddResult = new updateResultImpl()
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val resultIter = new Iterator[(String, (LoadMetadataDetails, ExecutionErrors))] {
- val partitionID = "0"
val loadMetadataDetails = new LoadMetadataDetails
val executionErrors = ExecutionErrors(FailureCauses.NONE, "")
var uniqueLoadStatusId = ""
@@ -658,10 +656,9 @@ object CarbonDataRDDFactory {
CarbonCommonConstants.UNDERSCORE +
(index + "_0")
- loadMetadataDetails.setPartitionCount(partitionID)
+ loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID)
loadMetadataDetails.setLoadName(segId)
loadMetadataDetails.setSegmentStatus(SegmentStatus.LOAD_FAILURE)
- carbonLoadModel.setPartitionId(partitionID)
carbonLoadModel.setSegmentId(segId)
carbonLoadModel.setTaskNo(String.valueOf(index))
carbonLoadModel.setFactTimeStamp(updateModel.get.updatedTimeStamp)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index 99e5732..945f47f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -105,7 +105,6 @@ with Serializable {
model,
conf
)
- model.setPartitionId("0")
model.setUseOnePass(options.getOrElse("onepass", "false").toBoolean)
model.setDictionaryServerHost(options.getOrElse("dicthost", null))
model.setDictionaryServerPort(options.getOrElse("dictport", "-1").toInt)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
index b5325ef..aadee81 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAlterPartitionTable.scala
@@ -858,7 +858,7 @@ class TestAlterPartitionTable extends QueryTest with BeforeAndAfterAll {
def getDataFiles(carbonTable: CarbonTable, segmentId: String): Array[CarbonFile] = {
val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
carbonTable.getTablePath)
- val segmentDir = tablePath.getCarbonDataDirectoryPath("0", segmentId)
+ val segmentDir = tablePath.getCarbonDataDirectoryPath(segmentId)
val carbonFile = FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir))
val dataFiles = carbonFile.listFiles(new CarbonFileFilter() {
override def accept(file: CarbonFile): Boolean = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
index 102df39..9da7244 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
@@ -173,6 +173,7 @@ class TableBucketingTestCase extends Spark2QueryTest with BeforeAndAfterAll {
case s: ShuffleExchange => shuffleExists = true
}
assert(!shuffleExists, "shuffle should not exist on bucket tables")
+ sql("DROP TABLE bucketed_parquet_table")
}
test("test create table with bucket join of carbon table and non bucket parquet table") {
@@ -197,6 +198,7 @@ class TableBucketingTestCase extends Spark2QueryTest with BeforeAndAfterAll {
case s: ShuffleExchange => shuffleExists = true
}
assert(shuffleExists, "shuffle should exist on non bucket tables")
+ sql("DROP TABLE parquet_table")
}
test("test scalar subquery with equal") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
index 7b1ab9d..e291f41 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
@@ -40,8 +40,6 @@ public class CarbonDataLoadConfiguration {
private String[] header;
- private String partitionId;
-
private String segmentId;
private String taskNo;
@@ -189,14 +187,6 @@ public class CarbonDataLoadConfiguration {
this.tableIdentifier = tableIdentifier;
}
- public String getPartitionId() {
- return partitionId;
- }
-
- public void setPartitionId(String partitionId) {
- this.partitionId = partitionId;
- }
-
public String getSegmentId() {
return segmentId;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index f7eff81..cf045a4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -152,7 +152,6 @@ public final class DataLoadProcessBuilder {
configuration.setTableIdentifier(identifier);
configuration.setSchemaUpdatedTimeStamp(carbonTable.getTableLastUpdatedTime());
configuration.setHeader(loadModel.getCsvHeaderColumns());
- configuration.setPartitionId(loadModel.getPartitionId());
configuration.setSegmentId(loadModel.getSegmentId());
configuration.setTaskNo(loadModel.getTaskNo());
configuration.setDataLoadProperty(DataLoadProcessorConstants.COMPLEX_DELIMITERS,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java b/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
index e2be79c..a8db6c9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/TableProcessingOperations.java
@@ -63,8 +63,7 @@ public class TableProcessingOperations {
//delete folder which metadata no exist in tablestatus
for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
- final String partitionCount = i + "";
- String partitionPath = carbonTablePath.getPartitionDir(partitionCount);
+ String partitionPath = carbonTablePath.getPartitionDir();
FileFactory.FileType fileType = FileFactory.getFileType(partitionPath);
if (FileFactory.isFileExist(partitionPath, fileType)) {
CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index d41455f..fef2da6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -39,8 +39,6 @@ public class CarbonLoadModel implements Serializable {
private String colDictFilePath;
- private String partitionId;
-
private CarbonDataLoadSchema carbonDataLoadSchema;
private boolean aggLoadRequest;
@@ -356,55 +354,6 @@ public class CarbonLoadModel implements Serializable {
}
/**
- * get copy with partition
- *
- * @param uniqueId
- * @return
- */
- public CarbonLoadModel getCopyWithPartition(String uniqueId) {
- CarbonLoadModel copy = new CarbonLoadModel();
- copy.tableName = tableName;
- copy.factFilePath = factFilePath + '/' + uniqueId;
- copy.databaseName = databaseName;
- copy.partitionId = uniqueId;
- copy.aggLoadRequest = aggLoadRequest;
- copy.loadMetadataDetails = loadMetadataDetails;
- copy.isRetentionRequest = isRetentionRequest;
- copy.complexDelimiterLevel1 = complexDelimiterLevel1;
- copy.complexDelimiterLevel2 = complexDelimiterLevel2;
- copy.carbonDataLoadSchema = carbonDataLoadSchema;
- copy.blocksID = blocksID;
- copy.taskNo = taskNo;
- copy.factTimeStamp = factTimeStamp;
- copy.segmentId = segmentId;
- copy.serializationNullFormat = serializationNullFormat;
- copy.badRecordsLoggerEnable = badRecordsLoggerEnable;
- copy.badRecordsAction = badRecordsAction;
- copy.escapeChar = escapeChar;
- copy.quoteChar = quoteChar;
- copy.commentChar = commentChar;
- copy.timestampformat = timestampformat;
- copy.dateFormat = dateFormat;
- copy.defaultTimestampFormat = defaultTimestampFormat;
- copy.maxColumns = maxColumns;
- copy.tablePath = tablePath;
- copy.useOnePass = useOnePass;
- copy.dictionaryServerHost = dictionaryServerHost;
- copy.dictionaryServerPort = dictionaryServerPort;
- copy.dictionaryServerSecretKey = dictionaryServerSecretKey;
- copy.dictionaryEncryptServerSecure = dictionaryEncryptServerSecure;
- copy.dictionaryServiceProvider = dictionaryServiceProvider;
- copy.preFetch = preFetch;
- copy.isEmptyDataBadRecord = isEmptyDataBadRecord;
- copy.skipEmptyLine = skipEmptyLine;
- copy.sortScope = sortScope;
- copy.batchSortSizeInMb = batchSortSizeInMb;
- copy.badRecordsLocation = badRecordsLocation;
- copy.isAggLoadRequest = isAggLoadRequest;
- return copy;
- }
-
- /**
* Get copy with taskNo.
* Broadcast value is shared in process, so we need to copy it to make sure the value in each
* task independently.
@@ -416,7 +365,6 @@ public class CarbonLoadModel implements Serializable {
copy.tableName = tableName;
copy.factFilePath = factFilePath;
copy.databaseName = databaseName;
- copy.partitionId = partitionId;
copy.aggLoadRequest = aggLoadRequest;
copy.loadMetadataDetails = loadMetadataDetails;
copy.isRetentionRequest = isRetentionRequest;
@@ -460,19 +408,15 @@ public class CarbonLoadModel implements Serializable {
/**
* get CarbonLoadModel with partition
*
- * @param uniqueId
- * @param filesForPartition
* @param header
* @param delimiter
* @return
*/
- public CarbonLoadModel getCopyWithPartition(String uniqueId, List<String> filesForPartition,
- String header, String delimiter) {
+ public CarbonLoadModel getCopyWithPartition(String header, String delimiter) {
CarbonLoadModel copyObj = new CarbonLoadModel();
copyObj.tableName = tableName;
copyObj.factFilePath = null;
copyObj.databaseName = databaseName;
- copyObj.partitionId = uniqueId;
copyObj.aggLoadRequest = aggLoadRequest;
copyObj.loadMetadataDetails = loadMetadataDetails;
copyObj.isRetentionRequest = isRetentionRequest;
@@ -514,20 +458,6 @@ public class CarbonLoadModel implements Serializable {
}
/**
- * @return the partitionId
- */
- public String getPartitionId() {
- return partitionId;
- }
-
- /**
- * @param partitionId the partitionId to set
- */
- public void setPartitionId(String partitionId) {
- this.partitionId = partitionId;
- }
-
- /**
* @param tablePath The tablePath to set.
*/
public void setTablePath(String tablePath) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
index 6432d38..fcc88b5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterImpl.java
@@ -73,8 +73,8 @@ public class ParallelReadMergeSorterImpl extends AbstractMergeSorter {
String[] storeLocations =
CarbonDataProcessorUtil.getLocalDataFolderLocation(
sortParameters.getDatabaseName(), sortParameters.getTableName(),
- String.valueOf(sortParameters.getTaskNo()), sortParameters.getPartitionID(),
- sortParameters.getSegmentId() + "", false, false);
+ String.valueOf(sortParameters.getTaskNo()), sortParameters.getSegmentId(),
+ false, false);
// Set the data file location
String[] dataFolderLocations = CarbonDataProcessorUtil.arrayAppend(storeLocations,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
index c7030dd..b7452a7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
@@ -133,10 +133,10 @@ public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorte
}
private SingleThreadFinalSortFilesMerger getFinalMerger(String bucketId) {
- String[] storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(sortParameters.getDatabaseName(), sortParameters.getTableName(),
- String.valueOf(sortParameters.getTaskNo()), bucketId,
- sortParameters.getSegmentId() + "", false, false);
+ String[] storeLocation = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ sortParameters.getDatabaseName(), sortParameters.getTableName(),
+ String.valueOf(sortParameters.getTaskNo()), sortParameters.getSegmentId(),
+ false, false);
// Set the data file location
String[] dataFolderLocation = CarbonDataProcessorUtil.arrayAppend(storeLocation, File.separator,
CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
@@ -181,10 +181,9 @@ public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorte
}
private void setTempLocation(SortParameters parameters) {
- String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(parameters.getDatabaseName(),
- parameters.getTableName(), parameters.getTaskNo(),
- parameters.getPartitionID(), parameters.getSegmentId(), false, false);
+ String[] carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ parameters.getDatabaseName(), parameters.getTableName(), parameters.getTaskNo(),
+ parameters.getSegmentId(), false, false);
String[] tmpLocs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator,
CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
parameters.setTempFileLocation(tmpLocs);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
index c5579d9..ed3a55d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeBatchParallelReadMergeSorterImpl.java
@@ -219,10 +219,9 @@ public class UnsafeBatchParallelReadMergeSorterImpl extends AbstractMergeSorter
}
private void setTempLocation(SortParameters parameters) {
- String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(parameters.getDatabaseName(),
- parameters.getTableName(), parameters.getTaskNo(), batchCount + "",
- parameters.getSegmentId(), false, false);
+ String[] carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ parameters.getDatabaseName(), parameters.getTableName(), parameters.getTaskNo(),
+ parameters.getSegmentId(), false, false);
String[] tempDirs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
parameters.setTempFileLocation(tempDirs);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3b4c51c0/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
index 3c48e4d..f605b22 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
@@ -119,18 +119,17 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
Iterator<CarbonRowBatch>[] batchIterator = new Iterator[bucketingInfo.getNumberOfBuckets()];
for (int i = 0; i < sortDataRows.length; i++) {
- batchIterator[i] =
- new MergedDataIterator(String.valueOf(i), batchSize, intermediateFileMergers[i]);
+ batchIterator[i] = new MergedDataIterator(batchSize, intermediateFileMergers[i]);
}
return batchIterator;
}
- private UnsafeSingleThreadFinalSortFilesMerger getFinalMerger(String bucketId) {
- String[] storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(sortParameters.getDatabaseName(), sortParameters.getTableName(),
- String.valueOf(sortParameters.getTaskNo()), bucketId,
- sortParameters.getSegmentId() + "", false, false);
+ private UnsafeSingleThreadFinalSortFilesMerger getFinalMerger() {
+ String[] storeLocation = CarbonDataProcessorUtil.getLocalDataFolderLocation(
+ sortParameters.getDatabaseName(), sortParameters.getTableName(),
+ String.valueOf(sortParameters.getTaskNo()), sortParameters.getSegmentId(),
+ false, false);
// Set the data file location
String[] dataFolderLocation = CarbonDataProcessorUtil.arrayAppend(storeLocation,
File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
@@ -173,7 +172,7 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
private void setTempLocation(SortParameters parameters) {
String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
.getLocalDataFolderLocation(parameters.getDatabaseName(), parameters.getTableName(),
- parameters.getTaskNo(), parameters.getPartitionID(), parameters.getSegmentId(),
+ parameters.getTaskNo(), parameters.getSegmentId(),
false, false);
String[] tmpLoc = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator,
CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
@@ -224,7 +223,6 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
private class MergedDataIterator extends CarbonIterator<CarbonRowBatch> {
- private String partitionId;
private int batchSize;
@@ -232,9 +230,8 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
private UnsafeIntermediateMerger intermediateMerger;
- public MergedDataIterator(String partitionId, int batchSize,
+ public MergedDataIterator(int batchSize,
UnsafeIntermediateMerger intermediateMerger) {
- this.partitionId = partitionId;
this.batchSize = batchSize;
this.intermediateMerger = intermediateMerger;
this.firstRow = true;
@@ -245,7 +242,7 @@ public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMerg
@Override public boolean hasNext() {
if (firstRow) {
firstRow = false;
- finalMerger = getFinalMerger(partitionId);
+ finalMerger = getFinalMerger();
List<UnsafeCarbonRowPage> rowPages = intermediateMerger.getRowPages();
finalMerger.startFinalMerge(rowPages.toArray(new UnsafeCarbonRowPage[rowPages.size()]),
intermediateMerger.getMergedPages());
[30/50] [abbrv] carbondata git commit: [CARBONDATA-1991] Select query
from a streaming table throws ClassCastException
Posted by ja...@apache.org.
[CARBONDATA-1991] Select query from a streaming table throws ClassCastException
This closes #1778
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/241b2657
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/241b2657
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/241b2657
Branch: refs/heads/carbonstore
Commit: 241b2657aea0d7ac111a34b2f01631b30a7eaa50
Parents: 90aeaa6
Author: anubhav100 <an...@knoldus.in>
Authored: Mon Jan 8 18:11:47 2018 +0530
Committer: QiangCai <qi...@qq.com>
Committed: Mon Jan 29 16:09:09 2018 +0800
----------------------------------------------------------------------
.../carbondata/hadoop/streaming/CarbonStreamRecordReader.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/241b2657/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
index 0086a3c..19626f0 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
@@ -699,7 +699,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
col.putByteArray(rowId, v.getBytes());
} else if (t instanceof org.apache.spark.sql.types.DecimalType) {
DecimalType dt = (DecimalType)t;
- Decimal d = (Decimal) value;
+ Decimal d = Decimal.fromDecimal(value);
if (dt.precision() <= Decimal.MAX_INT_DIGITS()) {
col.putInt(rowId, (int)d.toUnscaledLong());
} else if (dt.precision() <= Decimal.MAX_LONG_DIGITS()) {
[05/50] [abbrv] carbondata git commit: [CARBONDATA-1941]
Documentation added for Lock Retry
Posted by ja...@apache.org.
[CARBONDATA-1941] Documentation added for Lock Retry
The properties, default value, and description added for the lock retry
This closes #1780
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bc305c1e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bc305c1e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bc305c1e
Branch: refs/heads/carbonstore
Commit: bc305c1e11a6d6e22168f09ee7f879610bcf042c
Parents: 2d9de50
Author: Shahid <sh...@gmail.com>
Authored: Tue Jan 9 19:06:19 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Jan 19 14:50:19 2018 +0800
----------------------------------------------------------------------
docs/configuration-parameters.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc305c1e/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 338854f..fc10646 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -101,10 +101,10 @@ This section provides the details of all the configurations required for CarbonD
| carbon.csv.read.buffersize.byte | 1048576 | csv reading buffer size. |
| high.cardinality.value | 100000 | To identify and apply compression for non-high cardinality columns. |
| carbon.merge.sort.reader.thread | 3 | Maximum no of threads used for reading intermediate files for final merging. |
-| carbon.load.metadata.lock.retries | 3 | Maximum number of retries to get the metadata lock for loading data to table. |
-| carbon.load.metadata.lock.retry.timeout.sec | 5 | Interval between the retries to get the lock. |
| carbon.concurrent.lock.retries | 100 | Specifies the maximum number of retries to obtain the lock for concurrent operations. This is used for concurrent loading. |
| carbon.concurrent.lock.retry.timeout.sec | 1 | Specifies the interval between the retries to obtain the lock for concurrent operations. |
+| carbon.lock.retries | 3 | Specifies the maximum number of retries to obtain the lock for any operations other than load. |
+| carbon.lock.retry.timeout.sec | 5 | Specifies the interval between the retries to obtain the lock for any operation other than load. |
| carbon.tempstore.location | /opt/Carbon/TempStoreLoc | Temporary store location. By default it takes System.getProperty("java.io.tmpdir"). |
| carbon.load.log.counter | 500000 | Data loading records count logger. |
| carbon.skip.empty.line | false | Setting this property ignores the empty lines in the CSV file during the data load |
[11/50] [abbrv] carbondata git commit: [HOTFIX] Listeners not getting
registered to the bus in CarbonSessionState Implementations
Posted by ja...@apache.org.
[HOTFIX] Listeners not getting registered to the bus in CarbonSessionState Implementations
Problem: Listeners are not getting registered if you create a new implementation of CarbonSessionState and add it to spark using configuration. In this case CarbonSession would not be created and thus listeners are not registered.
Solution: Register listeners in CarbonSessionState instead of CarbonSession.
This closes #1821
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/937868d1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/937868d1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/937868d1
Branch: refs/heads/carbonstore
Commit: 937868d1b45af56c575ee77b93b99c35b8d632b7
Parents: 23bc051
Author: kunal642 <ku...@gmail.com>
Authored: Wed Jan 17 15:33:25 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Jan 19 19:16:59 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/datamap/TableDataMap.java | 4 +-
.../events/OperationEventListener.java | 21 +++++-
.../carbondata/events/OperationListenerBus.java | 7 +-
.../scala/org/apache/spark/sql/CarbonEnv.scala | 24 ++++++-
.../org/apache/spark/sql/CarbonSession.scala | 25 -------
.../datamap/CarbonCreateDataMapCommand.scala | 71 ++++----------------
.../datamap/CarbonDropDataMapCommand.scala | 19 ++----
.../CreatePreAggregateTableCommand.scala | 22 +++---
.../command/table/CarbonDropTableCommand.scala | 29 +++-----
.../src/main/spark2.1/CarbonSessionState.scala | 3 +
.../src/main/spark2.2/CarbonSessionState.scala | 3 +
11 files changed, 97 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 61d2243..9c84891 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -36,7 +36,7 @@ import org.apache.carbondata.events.OperationEventListener;
* DataMap at the table level, user can add any number of datamaps for one table. Depends
* on the filter condition it can prune the blocklets.
*/
-public final class TableDataMap implements OperationEventListener {
+public final class TableDataMap extends OperationEventListener {
private AbsoluteTableIdentifier identifier;
@@ -163,7 +163,7 @@ public final class TableDataMap implements OperationEventListener {
return dataMapFactory;
}
- @Override public void onEvent(Event event, OperationContext opContext) {
+ @Override public void onEvent(Event event, OperationContext opContext) throws Exception {
dataMapFactory.fireEvent(event);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/core/src/main/java/org/apache/carbondata/events/OperationEventListener.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/events/OperationEventListener.java b/core/src/main/java/org/apache/carbondata/events/OperationEventListener.java
index 7007f9b..f783b80 100644
--- a/core/src/main/java/org/apache/carbondata/events/OperationEventListener.java
+++ b/core/src/main/java/org/apache/carbondata/events/OperationEventListener.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.events;
/**
* Event listener interface which describes the possible events
*/
-public interface OperationEventListener {
+public abstract class OperationEventListener {
/**
* Called on a specified event occurrence
@@ -27,5 +27,22 @@ public interface OperationEventListener {
* @param event
* @param operationContext
*/
- void onEvent(Event event, OperationContext operationContext) throws Exception;
+ protected abstract void onEvent(Event event, OperationContext operationContext) throws Exception;
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || !(obj instanceof OperationEventListener)) {
+ return false;
+ }
+ return getComparisonName().equals(((OperationEventListener) obj).getComparisonName());
+ }
+
+ private String getComparisonName() {
+ return getClass().getName();
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java b/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java
index 321ddd5..53349b8 100644
--- a/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java
+++ b/core/src/main/java/org/apache/carbondata/events/OperationListenerBus.java
@@ -37,7 +37,7 @@ public class OperationListenerBus {
/**
* Event map to hold all listeners corresponding to an event
*/
- protected Map<String, List<OperationEventListener>> eventMap =
+ protected Map<String, CopyOnWriteArrayList<OperationEventListener>> eventMap =
new ConcurrentHashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
/**
@@ -57,12 +57,13 @@ public class OperationListenerBus {
OperationEventListener operationEventListener) {
String eventType = eventClass.getName();
- List<OperationEventListener> operationEventListeners = eventMap.get(eventType);
+ CopyOnWriteArrayList<OperationEventListener> operationEventListeners = eventMap.get(eventType);
if (null == operationEventListeners) {
operationEventListeners = new CopyOnWriteArrayList<>();
eventMap.put(eventType, operationEventListeners);
}
- operationEventListeners.add(operationEventListener);
+ // addIfAbsent will only add the listener if it is not already present in the List.
+ operationEventListeners.addIfAbsent(operationEventListener);
return INSTANCE;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 82fbefa..bbc3c2d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -21,6 +21,7 @@ import java.util.concurrent.ConcurrentHashMap
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
+import org.apache.spark.sql.execution.command.preaaggregate._
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesFunction
import org.apache.spark.sql.hive._
@@ -30,7 +31,8 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util._
-import org.apache.carbondata.events.{CarbonEnvInitPreEvent, OperationContext, OperationListenerBus}
+import org.apache.carbondata.events._
+import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePreExecutionEvent, LoadTablePreStatusUpdateEvent}
import org.apache.carbondata.spark.rdd.SparkReadSupport
import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
@@ -124,6 +126,26 @@ object CarbonEnv {
}
/**
+ * Method to initialize Listeners to their respective events in the OperationListenerBus.
+ */
+ def initListeners(): Unit = {
+ OperationListenerBus.getInstance()
+ .addListener(classOf[LoadTablePreStatusUpdateEvent], LoadPostAggregateListener)
+ .addListener(classOf[DeleteSegmentByIdPreEvent], PreAggregateDeleteSegmentByIdPreListener)
+ .addListener(classOf[DeleteSegmentByDatePreEvent], PreAggregateDeleteSegmentByDatePreListener)
+ .addListener(classOf[UpdateTablePreEvent], UpdatePreAggregatePreListener)
+ .addListener(classOf[DeleteFromTablePreEvent], DeletePreAggregatePreListener)
+ .addListener(classOf[DeleteFromTablePreEvent], DeletePreAggregatePreListener)
+ .addListener(classOf[AlterTableDropColumnPreEvent], PreAggregateDropColumnPreListener)
+ .addListener(classOf[AlterTableRenamePreEvent], PreAggregateRenameTablePreListener)
+ .addListener(classOf[AlterTableDataTypeChangePreEvent], PreAggregateDataTypeChangePreListener)
+ .addListener(classOf[AlterTableAddColumnPreEvent], PreAggregateAddColumnsPreListener)
+ .addListener(classOf[LoadTablePreExecutionEvent], LoadPreAggregateTablePreListener)
+ .addListener(classOf[AlterTableCompactionPreStatusUpdateEvent],
+ AlterPreAggregateTableCompactionPostListener)
+ }
+
+ /**
* Return carbon table instance from cache or by looking up table in `sparkSession`
*/
def getCarbonTable(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 34e37c5..c2c15fe 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -24,18 +24,13 @@ import org.apache.hadoop.conf.Configuration
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
import org.apache.spark.sql.SparkSession.Builder
-import org.apache.spark.sql.execution.command.preaaggregate._
import org.apache.spark.sql.execution.streaming.CarbonStreamingQueryListener
import org.apache.spark.sql.hive.execution.command.CarbonSetCommand
import org.apache.spark.sql.internal.{SessionState, SharedState}
import org.apache.spark.util.{CarbonReflectionUtils, Utils}
-import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, ThreadLocalSessionInfo}
-import org.apache.carbondata.events._
-import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePreExecutionEvent, LoadTablePreStatusUpdateEvent}
-import org.apache.carbondata.spark.util.CommonUtil
/**
* Session implementation for {org.apache.spark.sql.SparkSession}
@@ -76,10 +71,6 @@ class CarbonSession(@transient val sc: SparkContext,
new CarbonSession(sparkContext, Some(sharedState))
}
- if (existingSharedState.isEmpty) {
- CarbonSession.initListeners()
- }
-
}
object CarbonSession {
@@ -247,20 +238,4 @@ object CarbonSession {
ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)
}
- def initListeners(): Unit = {
- OperationListenerBus.getInstance()
- .addListener(classOf[LoadTablePreStatusUpdateEvent], LoadPostAggregateListener)
- .addListener(classOf[DeleteSegmentByIdPreEvent], PreAggregateDeleteSegmentByIdPreListener)
- .addListener(classOf[DeleteSegmentByDatePreEvent], PreAggregateDeleteSegmentByDatePreListener)
- .addListener(classOf[UpdateTablePreEvent], UpdatePreAggregatePreListener)
- .addListener(classOf[DeleteFromTablePreEvent], DeletePreAggregatePreListener)
- .addListener(classOf[DeleteFromTablePreEvent], DeletePreAggregatePreListener)
- .addListener(classOf[AlterTableDropColumnPreEvent], PreAggregateDropColumnPreListener)
- .addListener(classOf[AlterTableRenamePreEvent], PreAggregateRenameTablePreListener)
- .addListener(classOf[AlterTableDataTypeChangePreEvent], PreAggregateDataTypeChangePreListener)
- .addListener(classOf[AlterTableAddColumnPreEvent], PreAggregateAddColumnsPreListener)
- .addListener(classOf[LoadTablePreExecutionEvent], LoadPreAggregateTablePreListener)
- .addListener(classOf[AlterTableCompactionPreStatusUpdateEvent],
- AlterPreAggregateTableCompactionPostListener)
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 574c31a..8e00635 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -41,6 +41,8 @@ case class CarbonCreateDataMapCommand(
queryString: Option[String])
extends AtomicRunnableCommand {
+ var createPreAggregateTableCommands: Seq[CreatePreAggregateTableCommand] = _
+
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
// since streaming segment does not support building index and pre-aggregate yet,
// so streaming table does not support create datamap
@@ -53,29 +55,29 @@ case class CarbonCreateDataMapCommand(
if (dmClassName.equals("org.apache.carbondata.datamap.AggregateDataMapHandler") ||
dmClassName.equalsIgnoreCase("preaggregate")) {
val timeHierarchyString = dmproperties.get(CarbonCommonConstants.TIMESERIES_HIERARCHY)
- if (timeHierarchyString.isDefined) {
+ createPreAggregateTableCommands = if (timeHierarchyString.isDefined) {
val details = TimeSeriesUtil
.validateAndGetTimeSeriesHierarchyDetails(
timeHierarchyString.get)
val updatedDmProperties = dmproperties - CarbonCommonConstants.TIMESERIES_HIERARCHY
- details.foreach { f =>
+ details.map { f =>
CreatePreAggregateTableCommand(dataMapName + '_' + f._1,
tableIdentifier,
dmClassName,
updatedDmProperties,
queryString.get,
- Some(f._1)).processMetadata(sparkSession)
- }
- }
- else {
- CreatePreAggregateTableCommand(
+ Some(f._1))
+ }.toSeq
+ } else {
+ Seq(CreatePreAggregateTableCommand(
dataMapName,
tableIdentifier,
dmClassName,
dmproperties,
queryString.get
- ).processMetadata(sparkSession)
+ ))
}
+ createPreAggregateTableCommands.flatMap(_.processMetadata(sparkSession))
} else {
val dataMapSchema = new DataMapSchema(dataMapName, dmClassName)
dataMapSchema.setProperties(new java.util.HashMap[String, String](dmproperties.asJava))
@@ -90,32 +92,7 @@ case class CarbonCreateDataMapCommand(
override def processData(sparkSession: SparkSession): Seq[Row] = {
if (dmClassName.equals("org.apache.carbondata.datamap.AggregateDataMapHandler") ||
dmClassName.equalsIgnoreCase("preaggregate")) {
- val timeHierarchyString = dmproperties.get(CarbonCommonConstants.TIMESERIES_HIERARCHY)
- if (timeHierarchyString.isDefined) {
- val details = TimeSeriesUtil
- .validateAndGetTimeSeriesHierarchyDetails(
- timeHierarchyString.get)
- val updatedDmProperties = dmproperties - CarbonCommonConstants.TIMESERIES_HIERARCHY
- details.foreach { f =>
- CreatePreAggregateTableCommand(dataMapName + '_' + f._1,
- tableIdentifier,
- dmClassName,
- updatedDmProperties,
- queryString.get,
- Some(f._1)).processData(sparkSession)
- }
- Seq.empty
- }
- else {
- CreatePreAggregateTableCommand(
- dataMapName,
- tableIdentifier,
- dmClassName,
- dmproperties,
- queryString.get
- ).processData(sparkSession)
- Seq.empty
- }
+ createPreAggregateTableCommands.flatMap(_.processData(sparkSession))
} else {
Seq.empty
}
@@ -125,31 +102,7 @@ case class CarbonCreateDataMapCommand(
if (dmClassName.equals("org.apache.carbondata.datamap.AggregateDataMapHandler") ||
dmClassName.equalsIgnoreCase("preaggregate")) {
val timeHierarchyString = dmproperties.get(CarbonCommonConstants.TIMESERIES_HIERARCHY)
- if (timeHierarchyString.isDefined) {
- val details = TimeSeriesUtil
- .validateAndGetTimeSeriesHierarchyDetails(
- timeHierarchyString.get)
- val updatedDmProperties = dmproperties - CarbonCommonConstants.TIMESERIES_HIERARCHY
- details.foreach { f =>
- CreatePreAggregateTableCommand(dataMapName + '_' + f._1,
- tableIdentifier,
- dmClassName,
- updatedDmProperties,
- queryString.get,
- Some(f._1)).undoMetadata(sparkSession, exception)
- }
- Seq.empty
- }
- else {
- CreatePreAggregateTableCommand(
- dataMapName,
- tableIdentifier,
- dmClassName,
- dmproperties,
- queryString.get
- ).undoMetadata(sparkSession, exception)
- Seq.empty
- }
+ createPreAggregateTableCommands.flatMap(_.undoMetadata(sparkSession, exception))
} else {
Seq.empty
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index e545b0b..0ad4457 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -36,7 +36,6 @@ import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverte
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.events._
-
/**
* Drops the datamap and any related tables associated with the datamap
* @param dataMapName
@@ -80,7 +79,6 @@ case class CarbonDropDataMapCommand(
val dataMapSchema = carbonTable.get.getTableInfo.getDataMapSchemaList.asScala.zipWithIndex.
find(_._1.getDataMapName.equalsIgnoreCase(dataMapName))
if (dataMapSchema.isDefined) {
-
val operationContext = new OperationContext
val dropDataMapPreEvent =
DropDataMapPreEvent(
@@ -97,16 +95,13 @@ case class CarbonDropDataMapCommand(
carbonTable.get.getTableInfo,
dbName,
tableName))(sparkSession)
- if (dataMapSchema.isDefined) {
- if (dataMapSchema.get._1.getRelationIdentifier != null) {
- commandToRun = CarbonDropTableCommand(
- ifExistsSet = true,
- Some(dataMapSchema.get._1.getRelationIdentifier.getDatabaseName),
- dataMapSchema.get._1.getRelationIdentifier.getTableName,
- dropChildTable = true)
- commandToRun.processMetadata(sparkSession)
- }
- }
+ commandToRun = CarbonDropTableCommand(
+ ifExistsSet = true,
+ Some(dataMapSchema.get._1.getRelationIdentifier.getDatabaseName),
+ dataMapSchema.get._1.getRelationIdentifier.getTableName,
+ dropChildTable = true
+ )
+ commandToRun.processMetadata(sparkSession)
// fires the event after dropping datamap from main table schema
val dropDataMapPostEvent =
DropDataMapPostEvent(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index 933bf91..56f298a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
/**
@@ -49,6 +50,8 @@ case class CreatePreAggregateTableCommand(
timeSeriesFunction: Option[String] = None)
extends AtomicRunnableCommand {
+ var parentTable: CarbonTable = _
+
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val updatedQuery = new CarbonSpark2SqlParser().addPreAggFunction(queryString)
val df = sparkSession.sql(updatedQuery)
@@ -58,7 +61,7 @@ case class CreatePreAggregateTableCommand(
val tableProperties = mutable.Map[String, String]()
dmProperties.foreach(t => tableProperties.put(t._1, t._2))
- val parentTable = PreAggregateUtil.getParentCarbonTable(df.logicalPlan)
+ parentTable = PreAggregateUtil.getParentCarbonTable(df.logicalPlan)
assert(parentTable.getTableName.equalsIgnoreCase(parentTableIdentifier.table),
"Parent table name is different in select and create")
@@ -130,6 +133,10 @@ case class CreatePreAggregateTableCommand(
parentTableIdentifier.table,
childSchema,
sparkSession)
+ // After updating the parent carbon table with data map entry extract the latest table object
+ // to be used in further create process.
+ parentTable = CarbonEnv.getCarbonTable(parentTableIdentifier.database,
+ parentTableIdentifier.table)(sparkSession)
Seq.empty
}
@@ -146,30 +153,27 @@ case class CreatePreAggregateTableCommand(
override def processData(sparkSession: SparkSession): Seq[Row] = {
// load child table if parent table has existing segments
- val dbName = CarbonEnv.getDatabaseName(parentTableIdentifier.database)(sparkSession)
- val parentCarbonTable = CarbonEnv.getCarbonTable(Some(dbName),
- parentTableIdentifier.table)(sparkSession)
// This will be used to check if the parent table has any segments or not. If not then no
// need to fire load for pre-aggregate table. Therefore reading the load details for PARENT
// table.
- val loadAvailable = SegmentStatusManager.readLoadMetadata(parentCarbonTable.getMetaDataFilepath)
+ val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetaDataFilepath)
.nonEmpty
if (loadAvailable) {
val updatedQuery = if (timeSeriesFunction.isDefined) {
- val dataMap = parentCarbonTable.getTableInfo.getDataMapSchemaList.asScala
+ val dataMap = parentTable.getTableInfo.getDataMapSchemaList.asScala
.filter(p => p.getDataMapName
.equalsIgnoreCase(dataMapName)).head
.asInstanceOf[AggregationDataMapSchema]
PreAggregateUtil.createTimeSeriesSelectQueryFromMain(dataMap.getChildSchema,
- parentCarbonTable.getTableName,
- parentCarbonTable.getDatabaseName)
+ parentTable.getTableName,
+ parentTable.getDatabaseName)
} else {
queryString
}
// Passing segmentToLoad as * because we want to load all the segments into the
// pre-aggregate table even if the user has set some segments on the parent table.
PreAggregateUtil.startDataLoadForDataMap(
- parentCarbonTable,
+ parentTable,
tableIdentifier,
updatedQuery,
segmentToLoad = "*",
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
index 312e8b0..9c0eb57 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
@@ -43,7 +43,7 @@ case class CarbonDropTableCommand(
extends AtomicRunnableCommand {
var carbonTable: CarbonTable = _
- var childTables : Seq[CarbonTable] = Seq.empty
+ var childDropCommands : Seq[CarbonDropTableCommand] = Seq.empty
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
@@ -89,22 +89,24 @@ case class CarbonDropTableCommand(
// drop all child tables
val childSchemas = carbonTable.getTableInfo.getDataMapSchemaList
- childTables = childSchemas.asScala
+ childDropCommands = childSchemas.asScala
.filter(_.getRelationIdentifier != null)
.map { childSchema =>
val childTable =
CarbonEnv.getCarbonTable(
TableIdentifier(childSchema.getRelationIdentifier.getTableName,
Some(childSchema.getRelationIdentifier.getDatabaseName)))(sparkSession)
- CarbonDropTableCommand(
+ val dropCommand = CarbonDropTableCommand(
ifExistsSet = true,
Some(childSchema.getRelationIdentifier.getDatabaseName),
childSchema.getRelationIdentifier.getTableName,
dropChildTable = true
- ).processMetadata(sparkSession)
- childTable
+ )
+ dropCommand.carbonTable = childTable
+ dropCommand
}
- }
+ childDropCommands.foreach(_.processMetadata(sparkSession))
+ }
// fires the event after dropping main table
val dropTablePostEvent: DropTablePostEvent =
@@ -136,8 +138,8 @@ case class CarbonDropTableCommand(
}
override def processData(sparkSession: SparkSession): Seq[Row] = {
+ // clear driver side index and dictionary cache
if (carbonTable != null) {
- // clear driver side index and dictionary cache
ManageDictionaryAndBTree.clearBTreeAndDictionaryLRUCache(carbonTable)
// delete the table folder
val tablePath = carbonTable.getTablePath
@@ -146,18 +148,9 @@ case class CarbonDropTableCommand(
val file = FileFactory.getCarbonFile(tablePath, fileType)
CarbonUtil.deleteFoldersAndFilesSilent(file)
}
- if (carbonTable.hasDataMapSchema && childTables.nonEmpty) {
+ if (carbonTable.hasDataMapSchema && childDropCommands.nonEmpty) {
// drop all child tables
- childTables.foreach { childTable =>
- val carbonDropCommand = CarbonDropTableCommand(
- ifExistsSet = true,
- Some(childTable.getDatabaseName),
- childTable.getTableName,
- dropChildTable = true
- )
- carbonDropCommand.carbonTable = childTable
- carbonDropCommand.processData(sparkSession)
- }
+ childDropCommands.foreach(_.processData(sparkSession))
}
}
Seq.empty
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
index a1fa382..0fe0f96 100644
--- a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
@@ -75,6 +75,9 @@ class CarbonSessionCatalog(
env
}
+ // Initialize all listeners to the Operation bus.
+ CarbonEnv.initListeners()
+
/**
* This method will invalidate carbonrelation from cache if carbon table is updated in
* carbon catalog
http://git-wip-us.apache.org/repos/asf/carbondata/blob/937868d1/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
index 5046541..66a20ea 100644
--- a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
@@ -91,6 +91,9 @@ class CarbonSessionCatalog(
carbonEnv
}
+ // Initialize all listeners to the Operation bus.
+ CarbonEnv.initListeners()
+
private def refreshRelationFromCache(identifier: TableIdentifier): Boolean = {
var isRefreshed = false
[13/50] [abbrv] carbondata git commit:
[CARBONDATA-2045][PreAggregate]Fixed Pre Aggregate failure when specific
segment is set
Posted by ja...@apache.org.
[CARBONDATA-2045][PreAggregate]Fixed Pre Aggregate failure when specific segment is set
Fixed issue : Query from segment set is not effective when pre-aggregate table is present
This closes #1823
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/953efce4
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/953efce4
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/953efce4
Branch: refs/heads/carbonstore
Commit: 953efce4a9516704d72571a9300e1d67b1ca1880
Parents: b4dc866
Author: kumarvishal <ku...@gmail.com>
Authored: Wed Jan 17 17:57:21 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Jan 20 00:22:47 2018 +0530
----------------------------------------------------------------------
.../sql/hive/CarbonPreAggregateRules.scala | 295 ++++++++++---------
1 file changed, 155 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/953efce4/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index d629c2a..79cbe05 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -268,39 +268,43 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
l.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonRelation.
metaData.hasAggregateDataMapSchema =>
val carbonTable = getCarbonTable(l)
- val list = scala.collection.mutable.HashSet.empty[QueryColumn]
- val aggregateExpressions = scala.collection.mutable.HashSet.empty[AggregateExpression]
- val isValidPlan = extractQueryColumnsFromAggExpression(
- grExp,
- aggExp,
- carbonTable,
- list,
- aggregateExpressions)
- if(isValidPlan) {
- val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
- aggregateExpressions,
+ if(isSpecificSegmentNotPresent(carbonTable)) {
+ val list = scala.collection.mutable.HashSet.empty[QueryColumn]
+ val aggregateExpressions = scala.collection.mutable.HashSet.empty[AggregateExpression]
+ val isValidPlan = extractQueryColumnsFromAggExpression(
+ grExp,
+ aggExp,
carbonTable,
- agg)
- if(null != aggDataMapSchema && null!= childPlan) {
- val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
- val (updatedGroupExp, updatedAggExp, newChild, None) =
- getUpdatedExpressions(grExp,
- aggExp,
- child,
- None,
- aggDataMapSchema,
- attributes,
- childPlan,
- carbonTable,
- agg)
- Aggregate(updatedGroupExp,
- updatedAggExp,
- CarbonReflectionUtils
- .getSubqueryAlias(sparkSession,
- Some(alias1),
- CarbonReflectionUtils
- .getSubqueryAlias(sparkSession, Some(alias2), newChild, None),
- None))
+ list,
+ aggregateExpressions)
+ if (isValidPlan) {
+ val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
+ aggregateExpressions,
+ carbonTable,
+ agg)
+ if (null != aggDataMapSchema && null != childPlan) {
+ val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
+ val (updatedGroupExp, updatedAggExp, newChild, None) =
+ getUpdatedExpressions(grExp,
+ aggExp,
+ child,
+ None,
+ aggDataMapSchema,
+ attributes,
+ childPlan,
+ carbonTable,
+ agg)
+ Aggregate(updatedGroupExp,
+ updatedAggExp,
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession,
+ Some(alias1),
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession, Some(alias2), newChild, None),
+ None))
+ } else {
+ agg
+ }
} else {
agg
}
@@ -316,35 +320,39 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
l.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonRelation.
metaData.hasAggregateDataMapSchema =>
val carbonTable = getCarbonTable(l)
- val list = scala.collection.mutable.HashSet.empty[QueryColumn]
- val aggregateExpressions = scala.collection.mutable.HashSet.empty[AggregateExpression]
- val isValidPlan = extractQueryColumnsFromAggExpression(
- grExp,
- aggExp,
- carbonTable,
- list,
- aggregateExpressions)
- if(isValidPlan) {
- val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
- aggregateExpressions,
+ if(isSpecificSegmentNotPresent(carbonTable)) {
+ val list = scala.collection.mutable.HashSet.empty[QueryColumn]
+ val aggregateExpressions = scala.collection.mutable.HashSet.empty[AggregateExpression]
+ val isValidPlan = extractQueryColumnsFromAggExpression(
+ grExp,
+ aggExp,
carbonTable,
- agg)
- if(null != aggDataMapSchema && null!= childPlan) {
- val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
- val (updatedGroupExp, updatedAggExp, newChild, None) =
- getUpdatedExpressions(grExp,
- aggExp,
- child,
- None,
- aggDataMapSchema,
- attributes,
- childPlan,
- carbonTable,
- agg)
- Aggregate(updatedGroupExp,
- updatedAggExp,
- CarbonReflectionUtils
- .getSubqueryAlias(sparkSession, Some(alias), newChild, None))
+ list,
+ aggregateExpressions)
+ if (isValidPlan) {
+ val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
+ aggregateExpressions,
+ carbonTable,
+ agg)
+ if (null != aggDataMapSchema && null != childPlan) {
+ val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
+ val (updatedGroupExp, updatedAggExp, newChild, None) =
+ getUpdatedExpressions(grExp,
+ aggExp,
+ child,
+ None,
+ aggDataMapSchema,
+ attributes,
+ childPlan,
+ carbonTable,
+ agg)
+ Aggregate(updatedGroupExp,
+ updatedAggExp,
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession, Some(alias), newChild, None))
+ } else {
+ agg
+ }
} else {
agg
}
@@ -360,45 +368,47 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
l.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonRelation.
metaData.hasAggregateDataMapSchema =>
val carbonTable = getCarbonTable(l)
- val list = scala.collection.mutable.HashSet.empty[QueryColumn]
- val aggregateExpressions = scala.collection.mutable.HashSet.empty[AggregateExpression]
- var isValidPlan = extractQueryColumnsFromAggExpression(
- grExp,
- aggExp,
- carbonTable,
- list,
- aggregateExpressions)
-
- if (isValidPlan) {
- isValidPlan = !CarbonReflectionUtils.hasPredicateSubquery(expression)
- }
-
- // getting the columns from filter expression
- if (isValidPlan) {
- extractColumnFromExpression(expression, list, carbonTable, true)
- }
- if (isValidPlan) {
- val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
- aggregateExpressions,
+ if(isSpecificSegmentNotPresent(carbonTable)) {
+ val list = scala.collection.mutable.HashSet.empty[QueryColumn]
+ val aggregateExpressions = scala.collection.mutable.HashSet.empty[AggregateExpression]
+ var isValidPlan = extractQueryColumnsFromAggExpression(
+ grExp,
+ aggExp,
carbonTable,
- agg)
- if (null != aggDataMapSchema && null != childPlan) {
- val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
- val (updatedGroupExp, updatedAggExp, newChild, updatedFilterExpression) =
- getUpdatedExpressions(grExp,
- aggExp,
- child,
- Some(expression),
- aggDataMapSchema,
- attributes,
- childPlan,
- carbonTable,
- agg)
- Aggregate(updatedGroupExp,
- updatedAggExp,
- Filter(updatedFilterExpression.get,
- CarbonReflectionUtils
- .getSubqueryAlias(sparkSession, Some(alias), newChild, None)))
+ list,
+ aggregateExpressions)
+ if (isValidPlan) {
+ isValidPlan = !CarbonReflectionUtils.hasPredicateSubquery(expression)
+ }
+ // getting the columns from filter expression
+ if (isValidPlan) {
+ extractColumnFromExpression(expression, list, carbonTable, true)
+ }
+ if (isValidPlan) {
+ val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
+ aggregateExpressions,
+ carbonTable,
+ agg)
+ if (null != aggDataMapSchema && null != childPlan) {
+ val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
+ val (updatedGroupExp, updatedAggExp, newChild, updatedFilterExpression) =
+ getUpdatedExpressions(grExp,
+ aggExp,
+ child,
+ Some(expression),
+ aggDataMapSchema,
+ attributes,
+ childPlan,
+ carbonTable,
+ agg)
+ Aggregate(updatedGroupExp,
+ updatedAggExp,
+ Filter(updatedFilterExpression.get,
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession, Some(alias), newChild, None)))
+ } else {
+ agg
+ }
} else {
agg
}
@@ -415,49 +425,54 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
l.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonRelation.
metaData.hasAggregateDataMapSchema =>
val carbonTable = getCarbonTable(l)
- val list = scala.collection.mutable.HashSet.empty[QueryColumn]
- val aggregateExpressions = scala.collection.mutable.HashSet.empty[AggregateExpression]
- var isValidPlan = extractQueryColumnsFromAggExpression(
- grExp,
- aggExp,
- carbonTable,
- list,
- aggregateExpressions)
+ if(isSpecificSegmentNotPresent(carbonTable)) {
+ val list = scala.collection.mutable.HashSet.empty[QueryColumn]
+ val aggregateExpressions = scala.collection.mutable.HashSet.empty[AggregateExpression]
+ var isValidPlan = extractQueryColumnsFromAggExpression(
+ grExp,
+ aggExp,
+ carbonTable,
+ list,
+ aggregateExpressions)
- if (isValidPlan) {
- isValidPlan = !CarbonReflectionUtils.hasPredicateSubquery(expression)
- }
+ if (isValidPlan) {
+ isValidPlan = !CarbonReflectionUtils.hasPredicateSubquery(expression)
+ }
- // getting the columns from filter expression
- if (isValidPlan) {
- extractColumnFromExpression(expression, list, carbonTable, true)
- }
- if (isValidPlan) {
- val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
- aggregateExpressions,
- carbonTable,
- agg)
- if (null != aggDataMapSchema && null != childPlan) {
- val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
- val (updatedGroupExp, updatedAggExp, newChild, updatedFilterExpression) =
- getUpdatedExpressions(grExp,
- aggExp,
- child,
- Some(expression),
- aggDataMapSchema,
- attributes,
- childPlan,
- carbonTable,
- agg)
- Aggregate(updatedGroupExp,
- updatedAggExp,
- Filter(updatedFilterExpression.get,
- CarbonReflectionUtils
- .getSubqueryAlias(sparkSession,
- Some(alias1),
- CarbonReflectionUtils
- .getSubqueryAlias(sparkSession, Some(alias2), newChild, None),
- None)))
+ // getting the columns from filter expression
+ if (isValidPlan) {
+ extractColumnFromExpression(expression, list, carbonTable, true)
+ }
+
+ if (isValidPlan) {
+ val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
+ aggregateExpressions,
+ carbonTable,
+ agg)
+ if (null != aggDataMapSchema && null != childPlan) {
+ val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
+ val (updatedGroupExp, updatedAggExp, newChild, updatedFilterExpression) =
+ getUpdatedExpressions(grExp,
+ aggExp,
+ child,
+ Some(expression),
+ aggDataMapSchema,
+ attributes,
+ childPlan,
+ carbonTable,
+ agg)
+ Aggregate(updatedGroupExp,
+ updatedAggExp,
+ Filter(updatedFilterExpression.get,
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession,
+ Some(alias1),
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession, Some(alias2), newChild, None),
+ None)))
+ } else {
+ agg
+ }
} else {
agg
}
@@ -652,7 +667,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
* @param carbonTable parent table
* @return is specific segment is present in session params
*/
- def isSpecificSegmentPresent(carbonTable: CarbonTable) : Boolean = {
+ def isSpecificSegmentNotPresent(carbonTable: CarbonTable) : Boolean = {
val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
if (carbonSessionInfo != null) {
carbonSessionInfo.getSessionParams
[29/50] [abbrv] carbondata git commit: [CARBONDATA-2040] Add standard
partiton example and optimize partition test case
Posted by ja...@apache.org.
[CARBONDATA-2040] Add standard partiton example and optimize partition test case
Add standard partiton example and optimize partition test cases.
This closes #1817
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/90aeaa65
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/90aeaa65
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/90aeaa65
Branch: refs/heads/carbonstore
Commit: 90aeaa65c0ef281795fe5618ee82ff5182eb66f4
Parents: 5a82232
Author: chenliang613 <ch...@huawei.com>
Authored: Wed Jan 17 13:03:56 2018 +0800
Committer: manishgupta88 <to...@gmail.com>
Committed: Mon Jan 29 11:30:31 2018 +0530
----------------------------------------------------------------------
.../examples/StandardPartitionExample.scala | 130 +++++++++++++++++++
...andardPartitionTableCompactionTestCase.scala | 4 +-
.../StandardPartitionTableQueryTestCase.scala | 3 +
3 files changed, 135 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/90aeaa65/examples/spark2/src/main/scala/org/apache/carbondata/examples/StandardPartitionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StandardPartitionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StandardPartitionExample.scala
new file mode 100644
index 0000000..5a8e3f5
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StandardPartitionExample.scala
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.spark.sql.{SaveMode, SparkSession}
+
+/**
+ * This example is dynamic partition, same as spark partition.
+ */
+
+object StandardPartitionExample {
+
+ def main(args: Array[String]) {
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val testData = s"$rootPath/integration/spark-common-test/src/test/resources/partition_data.csv"
+ val spark = ExampleUtils.createCarbonSession("StandardPartitionExample")
+
+ spark.sparkContext.setLogLevel("ERROR")
+
+ // 1. simple usage for StandardPartition
+ spark.sql("DROP TABLE IF EXISTS partitiontable0")
+ spark.sql("""
+ | CREATE TABLE partitiontable0
+ | (id Int,
+ | vin String,
+ | phonenumber Long,
+ | area String,
+ | salary Int)
+ | PARTITIONED BY (country String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ spark.sql(s"""
+ LOAD DATA LOCAL INPATH '$testData' into table partitiontable0
+ """)
+
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM partitiontable0
+ """.stripMargin).show()
+
+ // 2.compare the performance : with partition VS without partition
+
+ // build test data, if set the data is larger than 100M, it will take 10+ mins.
+ import scala.util.Random
+ import spark.implicits._
+ val r = new Random()
+ val df = spark.sparkContext.parallelize(1 to 10 * 1000 * 1000)
+ .map(x => ("No." + r.nextInt(100000), "country" + x % 8, "city" + x % 50, x % 300))
+ .toDF("ID", "country", "city", "population")
+
+ // Create table without partition
+ df.write.format("carbondata")
+ .option("tableName", "withoutpartition")
+ .option("compress", "true")
+ .mode(SaveMode.Overwrite).save()
+
+ // Create table with partition
+ spark.sql("DROP TABLE IF EXISTS withpartition")
+ spark.sql("""
+ | CREATE TABLE withpartition
+ | (ID String,
+ | city String,
+ | population Int)
+ | PARTITIONED BY (country String)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ df.write.format("carbondata")
+ .option("tableName", "withpartition")
+ .option("compress", "true")
+ .mode(SaveMode.Overwrite).save()
+
+ // define time function
+ def time(code: => Unit): Double = {
+ val start = System.currentTimeMillis()
+ code
+ // return time in second
+ (System.currentTimeMillis() - start).toDouble / 1000
+ }
+
+ val time_without_partition = time {
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM withoutpartition WHERE country='country3'
+ """.stripMargin).count()
+ }
+
+ val time_with_partition = time {
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM withpartition WHERE country='country3'
+ """.stripMargin).count()
+ }
+ // scalastyle:off
+ println("time of without partition:" + time_without_partition.toString)
+ println("time of with partition:" + time_with_partition.toString)
+ // scalastyle:on
+
+ spark.sql("DROP TABLE IF EXISTS partitiontable0")
+ spark.sql("DROP TABLE IF EXISTS withoutpartition")
+ spark.sql("DROP TABLE IF EXISTS withpartition")
+
+ spark.close()
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/90aeaa65/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
index 295922d..22ebd80 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCompactionTestCase.scala
@@ -189,8 +189,8 @@ class StandardPartitionTableCompactionTestCase extends QueryTest with BeforeAndA
sql(s"""insert into staticpartitioncompaction PARTITION(deptname='software') select empno,doj,workgroupcategoryname,deptno,projectcode,projectjoindate,projectenddate,attendance,utilization,salary,workgroupcategory,empname,designation from originTable""")
}
sql("CLEAN FILES FOR TABLE staticpartitioncompaction").show()
- var segments = sql("SHOW SEGMENTS FOR TABLE staticpartitioncompaction")
- var segmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
+ val segments = sql("SHOW SEGMENTS FOR TABLE staticpartitioncompaction")
+ val segmentSequenceIds = segments.collect().map { each => (each.toSeq) (0) }
assert(segmentSequenceIds.size==1)
assert(segmentSequenceIds.contains("0.1"))
CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/90aeaa65/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
index db873d1..d1ef94c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
@@ -237,6 +237,9 @@ class StandardPartitionTableQueryTestCase extends QueryTest with BeforeAndAfterA
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE staticpartitionload partition(empname='ravi') OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
+ val frame = sql("select empno,empname,designation,workgroupcategory,workgroupcategoryname,deptno,projectjoindate,attendance,deptname,projectcode,utilization,salary,projectenddate,doj from staticpartitionload")
+ verifyPartitionInfo(frame, Seq("empname=ravi"))
+
}
test("Creation of partition table should fail if the colname in table schema and partition column is same even if both are case sensitive"){
[43/50] [abbrv] carbondata git commit: [CARBONDATA-1987] Make package
name and directory paths consistent;
remove duplicate file CarbonColumnValidator
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSqlConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSqlConf.scala b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSqlConf.scala
new file mode 100644
index 0000000..2128ffd
--- /dev/null
+++ b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSqlConf.scala
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.internal.SQLConf.buildConf
+
+import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * To initialize dynamic values default param
+ */
+class CarbonSQLConf(sparkSession: SparkSession) {
+
+ val carbonProperties = CarbonProperties.getInstance()
+
+ /**
+ * To initialize dynamic param defaults along with usage docs
+ */
+ def addDefaultCarbonParams(): Unit = {
+ val ENABLE_UNSAFE_SORT =
+ buildConf(CarbonCommonConstants.ENABLE_UNSAFE_SORT)
+ .doc("To enable/ disable unsafe sort.")
+ .booleanConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
+ val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
+ buildConf(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
+ .doc("To set carbon task distribution.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
+ val BAD_RECORDS_LOGGER_ENABLE =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
+ .doc("To enable/ disable carbon bad record logger.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants
+ .CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
+ val BAD_RECORDS_ACTION =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
+ .doc("To configure the bad records action.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
+ val IS_EMPTY_DATA_BAD_RECORD =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
+ .doc("Property to decide weather empty data to be considered bad/ good record.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT
+ .toBoolean)
+ val SORT_SCOPE =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE)
+ .doc("Property to specify sort scope.")
+ .stringConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
+ val BATCH_SORT_SIZE_INMB =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB)
+ .doc("Property to specify batch sort size in MB.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
+ val SINGLE_PASS =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS)
+ .doc("Property to enable/disable single_pass.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
+ val BAD_RECORD_PATH =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH)
+ .doc("Property to configure the bad record location.")
+ .stringConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ val GLOBAL_SORT_PARTITIONS =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS)
+ .doc("Property to configure the global sort partitions.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
+ CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
+ val DATEFORMAT =
+ buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
+ .doc("Property to configure data format for date type columns.")
+ .stringConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
+ val CARBON_INPUT_SEGMENTS = buildConf(
+ "carbon.input.segments.<database_name>.<table_name>")
+ .doc("Property to configure the list of segments to query.").stringConf
+ .createWithDefault(carbonProperties
+ .getProperty("carbon.input.segments.<database_name>.<table_name>", "*"))
+ }
+ /**
+ * to set the dynamic properties default values
+ */
+ def addDefaultCarbonSessionParams(): Unit = {
+ sparkSession.conf.set(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
+ sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
+ carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
+ CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cdfcd75..2399e26 100644
--- a/pom.xml
+++ b/pom.xml
@@ -494,6 +494,7 @@
<sourceDirectory>${basedir}/hadoop/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark2/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark2/src/main/java</sourceDirectory>
+ <sourceDirectory>${basedir}/integration/spark2/src/main/spark2.1</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common-test/src/main/scala</sourceDirectory>
@@ -503,7 +504,7 @@
<sourceDirectory>${basedir}/integration/presto/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/presto/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/streaming/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/streaming/src/main/java</sourceDirectory>
+ <sourceDirectory>${basedir}/streaming/src/main/scala</sourceDirectory>
</sourceDirectories>
</configuration>
</plugin>
@@ -534,7 +535,7 @@
<artifactId>coveralls-maven-plugin</artifactId>
<version>4.3.0</version>
<configuration>
- <repoToken>Q8mcBW8dw34wjNiPAMx0SFQidspIwHw8g</repoToken>
+ <repoToken>opPwqWW41vYppv6KISea3u1TJvE1ugJ5Y</repoToken>
<sourceEncoding>UTF-8</sourceEncoding>
<jacocoReports>
<jacocoReport>${basedir}/target/carbondata-coverage-report/carbondata-coverage-report.xml
@@ -546,6 +547,7 @@
<sourceDirectory>${basedir}/processing/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/hadoop/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark2/src/main/scala</sourceDirectory>
+ <sourceDirectory>${basedir}/integration/spark2/src/main/spark2.2</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark2/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/spark-common/src/main/java</sourceDirectory>
@@ -556,7 +558,7 @@
<sourceDirectory>${basedir}/integration/presto/src/main/scala</sourceDirectory>
<sourceDirectory>${basedir}/integration/presto/src/main/java</sourceDirectory>
<sourceDirectory>${basedir}/streaming/src/main/java</sourceDirectory>
- <sourceDirectory>${basedir}/streaming/src/main/java</sourceDirectory>
+ <sourceDirectory>${basedir}/streaming/src/main/scala</sourceDirectory>
</sourceDirectories>
</configuration>
</plugin>
[26/50] [abbrv] carbondata git commit: [CARBONDATA-2063][Tests] Fix
bugs in interdependent tests
Posted by ja...@apache.org.
[CARBONDATA-2063][Tests] Fix bugs in interdependent tests
Tests should not depend on each other, otherwise tests will fail in developer's IDE if they only run a single test at once.
This closes #1843
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e4096325
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e4096325
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e4096325
Branch: refs/heads/carbonstore
Commit: e4096325468aaba39dbf69c4bb1a72bf0e321f92
Parents: 842b9e5
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Mon Jan 22 16:14:38 2018 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Sun Jan 28 15:23:17 2018 +0800
----------------------------------------------------------------------
.../testsuite/dataload/TestLoadDataGeneral.scala | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e4096325/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index c122287..09ca9e5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -22,7 +22,7 @@ import java.math.BigDecimal
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.Row
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.BeforeAndAfterEach
import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.datastore.impl.FileFactory
@@ -32,16 +32,15 @@ import org.apache.spark.sql.test.util.QueryTest
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
+class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
- override def beforeAll {
+ override def beforeEach {
sql("DROP TABLE IF EXISTS loadtest")
sql(
"""
| CREATE TABLE loadtest(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
-
}
private def checkSegmentExists(
@@ -74,7 +73,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$testData' into table loadtest")
checkAnswer(
sql("SELECT COUNT(*) FROM loadtest"),
- Seq(Row(10))
+ Seq(Row(4))
)
}
@@ -83,7 +82,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$testData' into table loadtest")
checkAnswer(
sql("SELECT COUNT(*) FROM loadtest"),
- Seq(Row(14))
+ Seq(Row(4))
)
}
@@ -92,7 +91,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$testData' into table loadtest")
checkAnswer(
sql("SELECT COUNT(*) FROM loadtest"),
- Seq(Row(18))
+ Seq(Row(4))
)
}
@@ -101,7 +100,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$testData' into table loadtest options ('delimiter'='\\017')")
checkAnswer(
sql("SELECT COUNT(*) FROM loadtest"),
- Seq(Row(22))
+ Seq(Row(4))
)
}
@@ -243,7 +242,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select * from stale"), Row("k"))
}
- override def afterAll {
+ override def afterEach {
sql("DROP TABLE if exists loadtest")
sql("drop table if exists invalidMeasures")
CarbonProperties.getInstance()
[14/50] [abbrv] carbondata git commit: [CARBONDATA-2060] Fix insert
overwrite on partition table
Posted by ja...@apache.org.
[CARBONDATA-2060] Fix insert overwrite on partition table
Problem:
When insert overwrite is done on partition table with the table which has empty data, it was not overwriting.
Solution:
when insert OverWrite is fired on partition table from empty table, it should create new empty segment and it should delete old segments.
This closes #1838
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/315f41c1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/315f41c1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/315f41c1
Branch: refs/heads/carbonstore
Commit: 315f41c12557d9d8a355210082f096e1e6a19aca
Parents: 953efce
Author: akashrn5 <ak...@gmail.com>
Authored: Fri Jan 19 20:27:05 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Tue Jan 23 22:51:07 2018 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 20 ++++++-----
.../hadoop/api/CarbonOutputCommitter.java | 4 ++-
.../src/test/resources/partData.csv | 1 +
...tandardPartitionTableOverwriteTestCase.scala | 36 ++++++++++++++++++++
.../table/CarbonCreateTableCommand.scala | 6 ++--
.../spark/sql/hive/CarbonFileMetastore.scala | 7 ++--
6 files changed, 59 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/315f41c1/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index f1d474a..b1c0c30 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2260,15 +2260,17 @@ public final class CarbonUtil {
case S3:
Path path = new Path(segmentPath);
FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
- FileStatus[] fileStatuses = fs.listStatus(path);
- if (null != fileStatuses) {
- for (FileStatus dataAndIndexStatus : fileStatuses) {
- String pathName = dataAndIndexStatus.getPath().getName();
- if (pathName.endsWith(CarbonTablePath.getCarbonIndexExtension()) || pathName
- .endsWith(CarbonTablePath.getCarbonMergeIndexExtension())) {
- carbonIndexSize += dataAndIndexStatus.getLen();
- } else if (pathName.endsWith(CarbonTablePath.getCarbonDataExtension())) {
- carbonDataSize += dataAndIndexStatus.getLen();
+ if (fs.exists(path)) {
+ FileStatus[] fileStatuses = fs.listStatus(path);
+ if (null != fileStatuses) {
+ for (FileStatus dataAndIndexStatus : fileStatuses) {
+ String pathName = dataAndIndexStatus.getPath().getName();
+ if (pathName.endsWith(CarbonTablePath.getCarbonIndexExtension()) || pathName
+ .endsWith(CarbonTablePath.getCarbonMergeIndexExtension())) {
+ carbonIndexSize += dataAndIndexStatus.getLen();
+ } else if (pathName.endsWith(CarbonTablePath.getCarbonDataExtension())) {
+ carbonDataSize += dataAndIndexStatus.getLen();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/315f41c1/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index bc7c56f..eb18bbd 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -69,6 +69,8 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
boolean overwriteSet = CarbonTableOutputFormat.isOverwriteSet(context.getConfiguration());
CarbonLoadModel loadModel = CarbonTableOutputFormat.getLoadModel(context.getConfiguration());
CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(loadModel, overwriteSet);
+ CarbonLoaderUtil.checkAndCreateCarbonDataLocation(loadModel.getSegmentId(),
+ loadModel.getCarbonDataLoadSchema().getCarbonTable());
CarbonTableOutputFormat.setLoadModel(context.getConfiguration(), loadModel);
}
@@ -103,7 +105,7 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
long segmentSize = CarbonLoaderUtil
.addDataIndexSizeIntoMetaEntry(newMetaEntry, loadModel.getSegmentId(), carbonTable);
- if (segmentSize > 0) {
+ if (segmentSize > 0 || overwriteSet) {
String operationContextStr =
context.getConfiguration().get(
CarbonTableOutputFormat.OPERATION_CONTEXT,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/315f41c1/integration/spark-common-test/src/test/resources/partData.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/partData.csv b/integration/spark-common-test/src/test/resources/partData.csv
new file mode 100644
index 0000000..bd0b15c
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/partData.csv
@@ -0,0 +1 @@
+9000,CUST_NAME_00000,ACTIVE_EMUI_VERSION_00000,1970-01-01 01:00:03,1970-01-01 02:00:03,123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1
http://git-wip-us.apache.org/repos/asf/carbondata/blob/315f41c1/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
index 4104ea3..8d31134 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
@@ -174,6 +174,36 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
checkAnswer(sql("select count(*) from weather7 where month=1"), Seq(Row(2)))
}
+ test("test insert overwrite on dynamic partition") {
+ sql("CREATE TABLE uniqdata_hive_dynamic (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double, INTEGER_COLUMN1 int)ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
+ sql("CREATE TABLE uniqdata_string_dynamic(CUST_ID int,CUST_NAME String,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY(ACTIVE_EMUI_VERSION string) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('TABLE_BLOCKSIZE'= '256 MB')")
+ sql(s"LOAD DATA INPATH '$resourcesPath/partData.csv' into table uniqdata_string_dynamic partition(active_emui_version='abc') OPTIONS('FILEHEADER'='CUST_ID,CUST_NAME ,ACTIVE_EMUI_VERSION,DOB,DOJ, BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1, Double_COLUMN2,INTEGER_COLUMN1','BAD_RECORDS_ACTION'='FORCE')")
+ sql(s"LOAD DATA INPATH '$resourcesPath/partData.csv' into table uniqdata_string_dynamic partition(active_emui_version='abc') OPTIONS('FILEHEADER'='CUST_ID,CUST_NAME ,ACTIVE_EMUI_VERSION,DOB,DOJ, BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1, Double_COLUMN2,INTEGER_COLUMN1','BAD_RECORDS_ACTION'='FORCE')")
+ sql("insert overwrite table uniqdata_string_dynamic partition(active_emui_version='xxx') select CUST_ID, CUST_NAME,DOB,doj, bigint_column1, bigint_column2, decimal_column1, decimal_column2,double_column1, double_column2,integer_column1 from uniqdata_hive_dynamic limit 10")
+ assert(sql("select * from uniqdata_string_dynamic").collect().length == 2)
+ sql("insert overwrite table uniqdata_string_dynamic select CUST_ID, CUST_NAME,DOB,doj, bigint_column1, bigint_column2, decimal_column1, decimal_column2,double_column1, double_column2,integer_column1,ACTIVE_EMUI_VERSION from uniqdata_hive_dynamic limit 10")
+ checkAnswer(sql("select * from uniqdata_string_dynamic"), sql("select * from uniqdata_hive_dynamic"))
+ }
+
+ test("test insert overwrite on static partition") {
+ sql("CREATE TABLE uniqdata_hive_static (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double, INTEGER_COLUMN1 int)ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
+ sql("CREATE TABLE uniqdata_string_static(CUST_ID int,CUST_NAME String,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY(ACTIVE_EMUI_VERSION string) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('TABLE_BLOCKSIZE'= '256 MB')")
+ sql(s"LOAD DATA INPATH '$resourcesPath/partData.csv' into table uniqdata_string_static OPTIONS('FILEHEADER'='CUST_ID,CUST_NAME ,ACTIVE_EMUI_VERSION,DOB,DOJ, BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1, Double_COLUMN2,INTEGER_COLUMN1','BAD_RECORDS_ACTION'='FORCE')")
+ sql(s"LOAD DATA INPATH '$resourcesPath/partData.csv' into table uniqdata_string_static OPTIONS('FILEHEADER'='CUST_ID,CUST_NAME ,ACTIVE_EMUI_VERSION,DOB,DOJ, BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1, Double_COLUMN2,INTEGER_COLUMN1','BAD_RECORDS_ACTION'='FORCE')")
+ sql("insert overwrite table uniqdata_string_static partition(active_emui_version='xxx') select CUST_ID, CUST_NAME,DOB,doj, bigint_column1, bigint_column2, decimal_column1, decimal_column2,double_column1, double_column2,integer_column1 from uniqdata_hive_static limit 10")
+ assert(sql("select * from uniqdata_string_static").collect().length == 2)
+ sql("insert overwrite table uniqdata_string_static select CUST_ID, CUST_NAME,DOB,doj, bigint_column1, bigint_column2, decimal_column1, decimal_column2,double_column1, double_column2,integer_column1,active_emui_version from uniqdata_hive_static limit 10")
+ checkAnswer(sql("select * from uniqdata_string_static"), sql("select * from uniqdata_hive_static"))
+ }
+
+ test("overwrite whole partition table with empty data") {
+ sql("create table partitionLoadTable(name string, age int) PARTITIONED BY(address string) stored by 'carbondata'")
+ sql("insert into partitionLoadTable select 'abc',4,'def'")
+ sql("insert into partitionLoadTable select 'abd',5,'xyz'")
+ sql("create table noLoadTable (name string, age int, address string) stored by 'carbondata'")
+ sql("insert overwrite table partitionLoadTable select * from noLoadTable")
+ checkAnswer(sql("select * from partitionLoadTable"), sql("select * from noLoadTable"))
+ }
override def afterAll = {
dropTable
@@ -188,6 +218,12 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
sql("drop table if exists partitionallcompaction")
sql("drop table if exists weather6")
sql("drop table if exists weather7")
+ sql("drop table if exists uniqdata_hive_static")
+ sql("drop table if exists uniqdata_hive_dynamic")
+ sql("drop table if exists uniqdata_string_static")
+ sql("drop table if exists uniqdata_string_dynamic")
+ sql("drop table if exists partitionLoadTable")
+ sql("drop table if exists noLoadTable")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/315f41c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
index dac08e0..f38304e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala
@@ -122,10 +122,10 @@ case class CarbonCreateTableCommand(
CarbonEnv.getInstance(sparkSession).carbonMetastore
.dropTable(tableIdentifier)(sparkSession)
- val msg = s"Create table'$tableName' in database '$dbName' failed."
- LOGGER.audit(msg)
+ val msg = s"Create table'$tableName' in database '$dbName' failed"
+ LOGGER.audit(msg.concat(", ").concat(e.getMessage))
LOGGER.error(e, msg)
- CarbonException.analysisException(msg)
+ CarbonException.analysisException(msg.concat(", ").concat(e.getMessage))
}
}
val createTablePostExecutionEvent: CreateTablePostExecutionEvent =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/315f41c1/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index 744fbd8..0c52100 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -17,9 +17,9 @@
package org.apache.spark.sql.hive
+import java.io.IOException
import java.net.URI
import java.util.UUID
-import java.util.concurrent.atomic.AtomicLong
import scala.collection.mutable.ArrayBuffer
@@ -344,7 +344,10 @@ class CarbonFileMetastore extends CarbonMetaStore {
val schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath)
val fileType = FileFactory.getFileType(schemaMetadataPath)
if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
- FileFactory.mkdirs(schemaMetadataPath, fileType)
+ val isDirCreated = FileFactory.mkdirs(schemaMetadataPath, fileType)
+ if (!isDirCreated) {
+ throw new IOException(s"Failed to create the metadata directory $schemaMetadataPath")
+ }
}
val thriftWriter = new ThriftWriter(schemaFilePath, false)
thriftWriter.open(FileWriteOperation.OVERWRITE)
[06/50] [abbrv] carbondata git commit:
[CARBONDATA-2042][PreAggregate]Fixed data mismatch issue in case timeseries
Posted by ja...@apache.org.
[CARBONDATA-2042][PreAggregate]Fixed data mismatch issue in case timeseries
Problem: Year, Month, Day level timeseries table giving wrong result
Solution: Timeseries UDF is not able to convert data when hour is in 24 hours format
This closes #1820
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/aac7af73
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/aac7af73
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/aac7af73
Branch: refs/heads/carbonstore
Commit: aac7af7333aabd3b94e5e91c49f3f3d766103048
Parents: bc305c1
Author: kumarvishal <ku...@gmail.com>
Authored: Wed Jan 17 14:34:56 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Jan 19 12:34:19 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/preagg/TimeSeriesUDF.java | 4 ++--
.../src/test/resources/data_sort.csv | 21 ++++++++++++++++++++
.../timeseries/TestTimeseriesDataLoad.scala | 21 +++++++++++++++++++-
.../CreatePreAggregateTableCommand.scala | 14 ++++++++++++-
.../preaaggregate/PreAggregateListeners.scala | 11 +++++-----
.../preaaggregate/PreAggregateUtil.scala | 19 ++++++++++++++++--
6 files changed, 79 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/aac7af73/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java b/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java
index 3aa4190..df712de 100644
--- a/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java
+++ b/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java
@@ -91,13 +91,13 @@ public class TimeSeriesUDF {
calendar.set(Calendar.MILLISECOND, 0);
calendar.set(Calendar.SECOND, 0);
calendar.set(Calendar.MINUTE, 0);
- calendar.set(Calendar.HOUR, 0);
+ calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.DAY_OF_MONTH, 1);
break;
case YEAR:
calendar.set(Calendar.MONTH, 1);
calendar.set(Calendar.DAY_OF_YEAR, 1);
- calendar.set(Calendar.HOUR, 0);
+ calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
calendar.set(Calendar.MILLISECOND, 0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/aac7af73/integration/spark-common-test/src/test/resources/data_sort.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/data_sort.csv b/integration/spark-common-test/src/test/resources/data_sort.csv
new file mode 100644
index 0000000..dcf58a1
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/data_sort.csv
@@ -0,0 +1,21 @@
+3,1,Mikaa1,2015-01-01 11:00:00,2015-01-01 13:00:00,198,260
+3,3,Mikaa2,2015-01-02 12:00:00,2015-01-01 14:00:00,278,230
+3,2,Mikaa1,2015-01-03 13:00:00,2015-01-01 15:00:00,2556,1
+3,5,Mikaa2,2015-01-04 14:00:00,2015-01-01 16:00:00,640,254
+3,4,Mikaa,2015-01-05 15:00:00,2015-01-01 17:00:00,980,256
+2,10,Mikaa,2015-01-06 16:00:00,2015-01-01 18:00:00,1,2378
+2,1,Mikaa,2015-01-07 17:00:00,2015-01-01 19:00:00,96,234
+2,9,max,2015-01-08 18:00:00,2015-01-01 20:00:00,89,236
+2,10,max,2015-01-09 19:00:00,2015-01-01 21:00:00,198.36,239.2
+2,6,Mikaa,2015-01-10 20:00:00,2015-01-01 22:00:00,134.9,23.8
+2,10,Mikaa,2015-01-11 21:00:00,2015-01-01 23:00:00,156.5,252.8
+3,5,Mikaa,2015-01-11 22:00:00,2015-01-02 00:00:00,10.2,100.56
+13,4,Mikaa,2015-01-11 23:00:00,2015-01-02 00:00:00,10.2,100.56
+14,8,Mikaa,2015-01-12 00:00:00,2015-01-02 00:00:00,10.2,100.56
+15,1,Mikaa,2015-01-12 01:00:00,2015-01-02 00:00:00,10.2,100.56
+16,2,Mikaa,2015-01-12 02:00:00,2015-01-02 00:00:00,10.2,100.56
+2,6,Mikaa,2015-01-12 03:00:00,2015-01-02 00:00:00,10.2,100.56
+18,7,Mikaa,2015-01-12 04:00:00,2015-01-02 00:00:00,10.2,100.56
+19,5,Mikaa,2015-01-12 05:00:00,2015-01-02 00:00:00,10.2,100.56
+20,9,Mikaa,2015-01-12 06:00:00,2015-01-02 00:00:00,10.2,100.56
+3,17,Mikaa,2015-01-12 06:00:00,2015-01-02 00:00:00,10.2,100.56
http://git-wip-us.apache.org/repos/asf/carbondata/blob/aac7af73/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
index 6a0ea62..4aad06c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
@@ -25,16 +25,34 @@ import org.scalatest.{BeforeAndAfterAll, Ignore}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-@Ignore
class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
sql("drop table if exists mainTable")
+ sql("drop table if exists table_03")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
+ sql("CREATE TABLE table_03 (imei string,age int,mac string,productdate timestamp,updatedate timestamp,gamePointId double,contractid double ) STORED BY 'org.apache.carbondata.format'")
+ sql(s"LOAD DATA inpath '$resourcesPath/data_sort.csv' INTO table table_03 options ('DELIMITER'=',', 'QUOTECHAR'='','FILEHEADER'='imei,age,mac,productdate,updatedate,gamePointId,contractid')")
+ sql("create datamap ag1 on table table_03 using 'preaggregate' DMPROPERTIES ( 'timeseries.eventtime'='productdate','timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1')as select productdate,mac,sum(age) from table_03 group by productdate,mac")
+
+ }
+ test("test Year level timeseries data validation1 ") {
+ checkAnswer( sql("select count(*) from table_03_ag1_year"),
+ Seq(Row(4)))
+ }
+
+ test("test month level timeseries data validation1 ") {
+ checkAnswer( sql("select count(*) from table_03_ag1_month"),
+ Seq(Row(4)))
+ }
+
+ test("test day level timeseries data validation1 ") {
+ checkAnswer( sql("select count(*) from table_03_ag1_day"),
+ Seq(Row(12)))
}
test("test Year level timeseries data validation") {
@@ -89,5 +107,6 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
override def afterAll: Unit = {
sql("drop table if exists mainTable")
+ sql("drop table if exists table_03")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/aac7af73/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index 3e86233..8b11548 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
/**
@@ -151,12 +152,23 @@ case class CreatePreAggregateTableCommand(
val loadAvailable = SegmentStatusManager.readLoadMetadata(parentCarbonTable.getMetaDataFilepath)
.nonEmpty
if (loadAvailable) {
+ val updatedQuery = if (timeSeriesFunction.isDefined) {
+ val dataMap = parentCarbonTable.getTableInfo.getDataMapSchemaList.asScala
+ .filter(p => p.getDataMapName
+ .equalsIgnoreCase(dataMapName)).head
+ .asInstanceOf[AggregationDataMapSchema]
+ PreAggregateUtil.createTimeSeriesSelectQueryFromMain(dataMap.getChildSchema,
+ parentCarbonTable.getTableName,
+ parentCarbonTable.getDatabaseName)
+ } else {
+ queryString
+ }
// Passing segmentToLoad as * because we want to load all the segments into the
// pre-aggregate table even if the user has set some segments on the parent table.
PreAggregateUtil.startDataLoadForDataMap(
parentCarbonTable,
tableIdentifier,
- queryString,
+ updatedQuery,
segmentToLoad = "*",
validateSegments = true,
isOverwrite = false,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/aac7af73/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
index 17e2f2b..fce32ab 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
@@ -62,17 +62,18 @@ object LoadPostAggregateListener extends OperationEventListener {
} else {
// for timeseries rollup policy
val tableSelectedForRollup = PreAggregateUtil.getRollupDataMapNameForTimeSeries(list,
- dataMapSchema)
+ dataMapSchema)
+ list += dataMapSchema
// if non of the rollup data map is selected hit the maintable and prepare query
if (tableSelectedForRollup.isEmpty) {
PreAggregateUtil.createTimeSeriesSelectQueryFromMain(dataMapSchema.getChildSchema,
- parentTableName,
- databasename)
+ parentTableName,
+ databasename)
} else {
// otherwise hit the select rollup datamap schema
PreAggregateUtil.createTimeseriesSelectQueryForRollup(dataMapSchema.getChildSchema,
- tableSelectedForRollup.get,
- databasename)
+ tableSelectedForRollup.get,
+ databasename)
}
}
val isOverwrite =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/aac7af73/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index d77f2c2..cd19e3b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -669,6 +669,7 @@ object PreAggregateUtil {
val groupingExpressions = scala.collection.mutable.ArrayBuffer.empty[String]
val columns = tableSchema.getListOfColumns.asScala
.filter(f => !f.getColumnName.equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE))
+ .sortBy(_.getSchemaOrdinal)
columns.foreach { a =>
if (a.getAggFunction.nonEmpty) {
aggregateColumns += s"${a.getAggFunction match {
@@ -681,13 +682,21 @@ object PreAggregateUtil {
.getNonAggChildColBasedByParent(a.getParentColumnTableRelations.
get(0).getColumnName).getColumnName
} , '${ a.getTimeSeriesFunction }')"
+ aggregateColumns += s"timeseries(${
+ selectedDataMapSchema
+ .getNonAggChildColBasedByParent(a.getParentColumnTableRelations.
+ get(0).getColumnName).getColumnName
+ } , '${ a.getTimeSeriesFunction }')"
} else {
groupingExpressions += selectedDataMapSchema
.getNonAggChildColBasedByParent(a.getParentColumnTableRelations.
get(0).getColumnName).getColumnName
+ aggregateColumns += selectedDataMapSchema
+ .getNonAggChildColBasedByParent(a.getParentColumnTableRelations.
+ get(0).getColumnName).getColumnName
}
}
- s"select ${ groupingExpressions.mkString(",") },${ aggregateColumns.mkString(",")
+ s"select ${ aggregateColumns.mkString(",")
} from $databaseName.${selectedDataMapSchema.getChildSchema.getTableName } " +
s"group by ${ groupingExpressions.mkString(",") }"
}
@@ -707,6 +716,7 @@ object PreAggregateUtil {
val groupingExpressions = scala.collection.mutable.ArrayBuffer.empty[String]
val columns = tableSchema.getListOfColumns.asScala
.filter(f => !f.getColumnName.equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE))
+ .sortBy(_.getSchemaOrdinal)
columns.foreach {a =>
if (a.getAggFunction.nonEmpty) {
aggregateColumns +=
@@ -715,11 +725,16 @@ object PreAggregateUtil {
groupingExpressions +=
s"timeseries(${ a.getParentColumnTableRelations.get(0).getColumnName },'${
a.getTimeSeriesFunction}')"
+ aggregateColumns +=
+ s"timeseries(${ a.getParentColumnTableRelations.get(0).getColumnName },'${
+ a.getTimeSeriesFunction
+ }')"
} else {
groupingExpressions += a.getParentColumnTableRelations.get(0).getColumnName
+ aggregateColumns += a.getParentColumnTableRelations.get(0).getColumnName
}
}
- s"select ${ groupingExpressions.mkString(",") },${
+ s"select ${
aggregateColumns.mkString(",")
} from $databaseName.${ parentTableName } group by ${ groupingExpressions.mkString(",") }"
[04/50] [abbrv] carbondata git commit: [CARBONDATA-2053] Added events
for streaming
Posted by ja...@apache.org.
[CARBONDATA-2053] Added events for streaming
This PR contains the code to fire events during following two steps :
1. Before starting the stream and after the completion of stream
2. Before adding a batch and after batch is added during stream.
This closes #1832
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2d9de502
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2d9de502
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2d9de502
Branch: refs/heads/carbonstore
Commit: 2d9de502951d124e4b87fe6d38434f5dd05ae7c2
Parents: 543a903
Author: rahulforallp <ra...@knoldus.in>
Authored: Thu Jan 18 17:24:24 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Fri Jan 19 09:26:07 2018 +0530
----------------------------------------------------------------------
.../streaming/StreamSinkFactory.scala | 33 +++++++++++++++++---
.../streaming/CarbonAppendableStreamSink.scala | 25 +++++++++++++--
2 files changed, 50 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2d9de502/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index cecc18c..1c7be6a 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -32,7 +32,8 @@ import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.hadoop.streaming.CarbonStreamOutputFormat
+import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
+import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
@@ -51,15 +52,29 @@ object StreamSinkFactory {
parameters: Map[String, String]): Sink = {
validateParameters(parameters)
- // prepare the stream segment
- val segmentId = getStreamSegmentId(carbonTable)
// build load model
val carbonLoadModel = buildCarbonLoadModelForStream(
sparkSession,
hadoopConf,
carbonTable,
parameters,
- segmentId)
+ "")
+ // fire pre event before streamin is started
+ val operationContext = new OperationContext
+ val loadTablePreExecutionEvent = new LoadTablePreExecutionEvent(
+ carbonTable.getCarbonTableIdentifier,
+ carbonLoadModel,
+ carbonLoadModel.getFactFilePath,
+ false,
+ parameters.asJava,
+ null,
+ false
+ )
+ OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext)
+ // prepare the stream segment
+ val segmentId = getStreamSegmentId(carbonTable)
+ carbonLoadModel.setSegmentId(segmentId)
+
// start server if necessary
val server = startDictionaryServer(
sparkSession,
@@ -71,13 +86,21 @@ object StreamSinkFactory {
carbonLoadModel.setUseOnePass(false)
}
// default is carbon appended stream sink
- new CarbonAppendableStreamSink(
+ val carbonAppendableStreamSink = new CarbonAppendableStreamSink(
sparkSession,
carbonTable,
segmentId,
parameters,
carbonLoadModel,
server)
+
+ // fire post event before streamin is started
+ val loadTablePostExecutionEvent = new LoadTablePostExecutionEvent(
+ carbonTable.getCarbonTableIdentifier,
+ carbonLoadModel
+ )
+ OperationListenerBus.getInstance().fireEvent(loadTablePostExecutionEvent, operationContext)
+ carbonAppendableStreamSink
}
private def validateParameters(parameters: Map[String, String]): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2d9de502/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
index ce9446f..849bf99 100644
--- a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
+++ b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala
@@ -19,13 +19,12 @@ package org.apache.spark.sql.execution.streaming
import java.util.Date
+import scala.collection.JavaConverters._
+
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce._
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
-import org.apache.hadoop.mapreduce.v2.api.records.JobId
-import org.apache.spark._
import org.apache.spark.TaskContext
-import org.apache.spark.internal.io._
import org.apache.spark.internal.io.FileCommitProtocol
import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage
import org.apache.spark.sql.{DataFrame, SparkSession}
@@ -42,8 +41,10 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.stats.QueryStatistic
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.hadoop.streaming.CarbonStreamOutputFormat
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
+import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.streaming.{CarbonStreamException, StreamHandoffRDD}
import org.apache.carbondata.streaming.parser.CarbonStreamParser
@@ -93,6 +94,18 @@ class CarbonAppendableStreamSink(
val statistic = new QueryStatistic()
+ // fire pre event on every batch add
+ val operationContext = new OperationContext
+ val loadTablePreExecutionEvent = new LoadTablePreExecutionEvent(
+ carbonTable.getCarbonTableIdentifier,
+ carbonLoadModel,
+ carbonLoadModel.getFactFilePath,
+ false,
+ parameters.asJava,
+ null,
+ false
+ )
+ OperationListenerBus.getInstance().fireEvent(loadTablePreExecutionEvent, operationContext)
checkOrHandOffSegment()
// committer will record how this spark job commit its output
@@ -119,6 +132,12 @@ class CarbonAppendableStreamSink(
hadoopConf,
carbonLoadModel,
server)
+ // fire post event on every batch add
+ val loadTablePostExecutionEvent = new LoadTablePostExecutionEvent(
+ carbonTable.getCarbonTableIdentifier,
+ carbonLoadModel
+ )
+ OperationListenerBus.getInstance().fireEvent(loadTablePostExecutionEvent, operationContext)
statistic.addStatistics(s"add batch: $batchId", System.currentTimeMillis())
CarbonAppendableStreamSink.LOGGER.info(
[46/50] [abbrv] carbondata git commit: [CARBONDATA-1948] Update help
document for the change made for CARBONDATA-1929
Posted by ja...@apache.org.
[CARBONDATA-1948] Update help document for the change made for CARBONDATA-1929
Update help document for the change made for CARBONDATA-1929
This closes #1884
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8314ea21
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8314ea21
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8314ea21
Branch: refs/heads/carbonstore
Commit: 8314ea21a2c9bed59e86864dda055be8be6011e3
Parents: c9a501d
Author: Varun Saxena <va...@apache.org>
Authored: Tue Jan 30 16:30:36 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Jan 31 12:05:10 2018 +0800
----------------------------------------------------------------------
docs/configuration-parameters.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8314ea21/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index 5794f4d..367d045 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -96,9 +96,9 @@ This section provides the details of all the configurations required for CarbonD
| Parameter | Default Value | Description |
|---------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| carbon.sort.file.write.buffer.size | 10485760 | File write buffer size used during sorting. |
+| carbon.sort.file.write.buffer.size | 16777216 | File write buffer size used during sorting (minValue = 10 KB, maxValue=10MB). |
| carbon.lock.type | LOCALLOCK | This configuration specifies the type of lock to be acquired during concurrent operations on table. There are following types of lock implementation: - LOCALLOCK: Lock is created on local file system as file. This lock is useful when only one spark driver (thrift server) runs on a machine and no other CarbonData spark application is launched concurrently. - HDFSLOCK: Lock is created on HDFS file system as file. This lock is useful when multiple CarbonData spark applications are launched and no ZooKeeper is running on cluster and HDFS supports file based locking. |
-| carbon.sort.intermediate.files.limit | 20 | Minimum number of intermediate files after which merged sort can be started. |
+| carbon.sort.intermediate.files.limit | 20 | Minimum number of intermediate files after which merged sort can be started (minValue = 2, maxValue=50). |
| carbon.block.meta.size.reserved.percentage | 10 | Space reserved in percentage for writing block meta data in CarbonData file. |
| carbon.csv.read.buffersize.byte | 1048576 | csv reading buffer size. |
| high.cardinality.value | 100000 | To identify and apply compression for non-high cardinality columns. |
[19/50] [abbrv] carbondata git commit: [CI] Enabled pre aggregate
tests and fixed insert query CI random query failure
Posted by ja...@apache.org.
[CI] Enabled pre aggregate tests and fixed insert query CI random query failure
Problem:
There is an issue of random null pointer exception from TaskContext while doing insert queries in CI that's why pre-aggregate tests are commented
Solution :
>From my analysis, It is because of lack of synchronization in Sparks TaskContext and same is rectified in Spark 2.2.
To fix this issue added a mock to TaskContext and ignored the exceptions in tests.
This closes #1854
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7d434426
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7d434426
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7d434426
Branch: refs/heads/carbonstore
Commit: 7d4344268e8ccbc415e116ff7babbd8cee7e5384
Parents: d9bb647
Author: ravipesala <ra...@gmail.com>
Authored: Wed Jan 24 09:17:07 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Jan 25 17:38:20 2018 +0800
----------------------------------------------------------------------
integration/spark-common-test/pom.xml | 4 ++
.../preaggregate/TestPreAggregateLoad.scala | 6 ++-
.../timeseries/TestTimeseriesCompaction.scala | 3 +-
.../timeseries/TestTimeseriesDataLoad.scala | 2 +
.../TestTimeseriesTableSelection.scala | 2 +
.../InsertIntoCarbonTableTestCase.scala | 2 +-
.../testsuite/sortcolumns/TestSortColumns.scala | 6 +--
.../org/apache/spark/util/SparkUtil4Test.scala | 55 +++++++++++++++++++-
8 files changed, 72 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d434426/integration/spark-common-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index b8d7c05..e3c7bd8 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -113,6 +113,10 @@
<version>2.2.1</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.jmockit</groupId>
+ <artifactId>jmockit</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d434426/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
index 0601099..4ebf150 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
@@ -19,17 +19,18 @@ package org.apache.carbondata.integration.spark.testsuite.preaggregate
import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
import org.scalatest.{BeforeAndAfterAll, Ignore}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-@Ignore
class TestPreAggregateLoad extends QueryTest with BeforeAndAfterAll {
val testData = s"$resourcesPath/sample.csv"
override def beforeAll(): Unit = {
+ SparkUtil4Test.createTaskMockUp(sqlContext)
sql("DROP TABLE IF EXISTS maintable")
}
@@ -304,8 +305,9 @@ test("check load and select for avg double datatype") {
sql("create table maintbl(year int,month int,name string,salary float) stored by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23','sort_columns'='month,year,name')")
sql("insert into maintbl select 10,11,'babu',12")
sql("insert into maintbl select 10,11,'babu',12")
+ val rows = sql("select name,avg(salary) from maintbl group by name").collect()
sql("create datamap maintbl_douoble on table maintbl using 'preaggregate' as select name,avg(salary) from maintbl group by name")
- checkAnswer(sql("select name,avg(salary) from maintbl group by name"), Row("babu", 12.89))
+ checkAnswer(sql("select name,avg(salary) from maintbl group by name"), rows)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d434426/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
index 561e640..a410fe4 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
@@ -17,17 +17,18 @@
package org.apache.carbondata.integration.spark.testsuite.timeseries
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
import org.scalatest.{BeforeAndAfterAll, Ignore}
import org.scalatest.Matchers._
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-@Ignore
class TestTimeseriesCompaction extends QueryTest with BeforeAndAfterAll {
var isCompactionEnabled = false
override def beforeAll: Unit = {
+ SparkUtil4Test.createTaskMockUp(sqlContext)
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d434426/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
index 4aad06c..d25710c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
@@ -20,6 +20,7 @@ import java.sql.Timestamp
import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
import org.scalatest.{BeforeAndAfterAll, Ignore}
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -28,6 +29,7 @@ import org.apache.carbondata.core.util.CarbonProperties
class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
+ SparkUtil4Test.createTaskMockUp(sqlContext)
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
sql("drop table if exists mainTable")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d434426/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
index 725ac24..a9d3965 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
@@ -21,12 +21,14 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
import org.scalatest.BeforeAndAfterAll
class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
+ SparkUtil4Test.createTaskMockUp(sqlContext)
sql("drop table if exists mainTable")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d434426/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index c9c8a59..6739c6c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -387,7 +387,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
private def checkSegment(tableName: String) : Boolean ={
- val storePath_t1 = metastoredb + s"/warehouse/${tableName.toLowerCase()}/Fact/Part0"
+ val storePath_t1 = s"$storeLocation/${tableName.toLowerCase()}/Fact/Part0"
val carbonFile_t1: CarbonFile = FileFactory
.getCarbonFile(storePath_t1, FileFactory.getFileType(storePath_t1))
var exists: Boolean = carbonFile_t1.exists()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d434426/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index f076324..7c288b3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -23,10 +23,12 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.util.SparkUtil4Test
class TestSortColumns extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ SparkUtil4Test.createTaskMockUp(sqlContext)
dropTable
sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
@@ -334,9 +336,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
assert(exceptionCaught.getMessage.equals("SORT_COLUMNS Either having duplicate columns : empno or it contains illegal argumnet."))
}
- // This testcase cause CI random failure, the reported error in CI is "TaskCompletionListener is null"
- // TODO: need to further analyze this.
- ignore("Test tableTwo data") {
+ test("Test tableTwo data") {
sql("insert into table tableTwo select id, count(age) from tableOne group by id")
checkAnswer(
sql("select id,age from tableTwo order by id"),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7d434426/integration/spark-common-test/src/test/scala/org/apache/spark/util/SparkUtil4Test.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/spark/util/SparkUtil4Test.scala b/integration/spark-common-test/src/test/scala/org/apache/spark/util/SparkUtil4Test.scala
index ef05264..163a88c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/spark/util/SparkUtil4Test.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/spark/util/SparkUtil4Test.scala
@@ -17,12 +17,17 @@
package org.apache.spark.util
-import org.apache.spark.SparkConf
+import mockit.{Invocation, Mock, MockUp}
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.{SparkConf, TaskContextImpl}
/**
* This class is for accessing utils in spark package for tests
*/
object SparkUtil4Test {
+
+ private var intializedMock = false
+
def getConfiguredLocalDirs(conf: SparkConf): Array[String] = {
Utils.getConfiguredLocalDirs(conf)
}
@@ -30,4 +35,52 @@ object SparkUtil4Test {
def getOrCreateLocalRootDirs(conf: SparkConf): Array[String] = {
Utils.getOrCreateLocalRootDirs(conf)
}
+
+ /**
+ * Creates the mock for TaskContextImpl to catch the exception and ignore it for CI.
+ * @param sqlContext
+ */
+ def createTaskMockUp(sqlContext: SQLContext): Unit = {
+ if (!intializedMock) {
+ if (sqlContext.sparkContext.version.startsWith("2.1")) {
+ createTaskMockUp2_1
+ } else if (sqlContext.sparkContext.version.startsWith("2.2")) {
+ createTaskMockUp2_2()
+ }
+ intializedMock = true
+ }
+ }
+
+ private def createTaskMockUp2_1 = {
+ new MockUp[TaskContextImpl] {
+ @Mock private[spark] def markTaskCompleted(invocation: Invocation): Unit = {
+ try {
+ invocation.proceed()
+ } catch {
+ case e: Exception => //ignore
+ }
+ }
+
+ @Mock def addTaskCompletionListener(invocation: Invocation, listener: TaskCompletionListener): TaskContextImpl = {
+ try {
+ invocation.proceed(listener)
+ } catch {
+ case e: Exception => // ignore
+ invocation.getInvokedInstance[TaskContextImpl]
+ }
+ }
+ }
+ }
+
+ private def createTaskMockUp2_2(): Unit = {
+ new MockUp[TaskContextImpl] {
+ @Mock private[spark] def markTaskCompleted(invocation: Invocation, error: Option[Throwable]): Unit = {
+ try {
+ invocation.proceed(error)
+ } catch {
+ case e: Exception => //ignore
+ }
+ }
+ }
+ }
}
[12/50] [abbrv] carbondata git commit: [CARBONDATA-2036] Fix the
insert static partition with integer values prefix with 0 not working
Posted by ja...@apache.org.
[CARBONDATA-2036] Fix the insert static partition with integer values prefix with 0 not working
When trying to insert overwrite on the static partition with 0 at first on int column has an issue.Example :create table test(d1 string) partition by (c1 int, c2 int, c3 int)And use insert overwrite table partition(01, 02, 03) select s1The above case has a problem as 01 is not converting to an actual integer to partition map file.Solution :Convert the partition values to corresponding datatype value before adding to partition file.
This closes #1833
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b4dc866f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b4dc866f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b4dc866f
Branch: refs/heads/carbonstore
Commit: b4dc866fec0a42196435c6da0e1413dc2a2398d1
Parents: 937868d
Author: ravipesala <ra...@gmail.com>
Authored: Thu Jan 18 18:51:50 2018 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Sat Jan 20 00:14:24 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 38 +++-
.../core/indexstore/UnsafeMemoryDMStore.java | 22 ++-
.../carbondata/core/util/CarbonProperties.java | 52 ++---
.../core/CarbonPropertiesValidationTest.java | 2 +-
...tandardPartitionTableOverwriteTestCase.scala | 20 ++
.../carbondata/spark/rdd/CarbonScanRDD.scala | 189 ++++++++++---------
.../carbondata/spark/util/CarbonScalaUtil.scala | 10 +-
.../management/CarbonLoadDataCommand.scala | 29 ++-
.../CarbonProjectForUpdateCommand.scala | 2 +-
.../datasources/CarbonFileFormat.scala | 61 ++----
.../apache/spark/sql/hive/CarbonMetaStore.scala | 4 +-
.../src/main/spark2.1/CarbonSQLConf.scala | 12 +-
.../src/main/spark2.2/CarbonSqlConf.scala | 12 +-
13 files changed, 246 insertions(+), 207 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index cd7abe0..13c8a42 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1299,11 +1299,6 @@ public final class CarbonCommonConstants {
@CarbonProperty
public static final String CARBON_CUSTOM_BLOCK_DISTRIBUTION = "carbon.custom.block.distribution";
- public static final String CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT = "false";
-
- @CarbonProperty
- public static final String CARBON_COMBINE_SMALL_INPUT_FILES = "carbon.mergeSmallFileRead.enable";
- public static final String CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT = "false";
public static final int DICTIONARY_DEFAULT_CARDINALITY = 1;
@CarbonProperty
@@ -1404,9 +1399,38 @@ public final class CarbonCommonConstants {
public static final String USE_DISTRIBUTED_DATAMAP_DEFAULT = "false";
- public static final String CARBON_USE_BLOCKLET_DISTRIBUTION = "carbon.blocklet.distribution";
+ /**
+ * This property defines how the tasks are splitted/combined and launch spark tasks during query
+ */
+ @CarbonProperty
+ public static final String CARBON_TASK_DISTRIBUTION = "carbon.task.distribution";
+
+ /**
+ * It combines the available blocks as per the maximum available tasks in the cluster.
+ */
+ public static final String CARBON_TASK_DISTRIBUTION_CUSTOM = "custom";
+
+ /**
+ * It creates the splits as per the number of blocks/carbondata files available for query.
+ */
+ public static final String CARBON_TASK_DISTRIBUTION_BLOCK = "block";
+
+ /**
+ * It creates the splits as per the number of blocklets available for query.
+ */
+ public static final String CARBON_TASK_DISTRIBUTION_BLOCKLET = "blocklet";
+
+ /**
+ * It merges all the small files and create tasks as per the configurable partition size.
+ */
+ public static final String CARBON_TASK_DISTRIBUTION_MERGE_FILES = "merge_small_files";
+
+ /**
+ * Default task distribution.
+ */
+ public static final String CARBON_TASK_DISTRIBUTION_DEFAULT = CARBON_TASK_DISTRIBUTION_BLOCK;
+
- public static final String CARBON_USE_BLOCKLET_DISTRIBUTION_DEFAULT = "true";
/**
* The property to configure the mdt file folder path, earlier it was pointing to the
* fixed carbon store path. This is needed in case of the federation setup when user removes
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
index dc630ff..31ecac2 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
@@ -36,7 +36,7 @@ public class UnsafeMemoryDMStore {
private MemoryBlock memoryBlock;
- private static int capacity = 8 * 1024 * 1024;
+ private static int capacity = 8 * 1024;
private int allocatedSize;
@@ -66,14 +66,8 @@ public class UnsafeMemoryDMStore {
* @param rowSize
*/
private void ensureSize(int rowSize) throws MemoryException {
- if (runningLength + rowSize >= allocatedSize) {
- MemoryBlock allocate =
- UnsafeMemoryManager.allocateMemoryWithRetry(taskId, allocatedSize + capacity);
- getUnsafe().copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
- allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
- UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
- allocatedSize = allocatedSize + capacity;
- memoryBlock = allocate;
+ while (runningLength + rowSize >= allocatedSize) {
+ increaseMemory();
}
if (this.pointers.length <= rowCount + 1) {
int[] newPointer = new int[pointers.length + 1000];
@@ -82,6 +76,16 @@ public class UnsafeMemoryDMStore {
}
}
+ private void increaseMemory() throws MemoryException {
+ MemoryBlock allocate =
+ UnsafeMemoryManager.allocateMemoryWithRetry(taskId, allocatedSize + capacity);
+ getUnsafe().copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
+ allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
+ UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
+ allocatedSize = allocatedSize + capacity;
+ memoryBlock = allocate;
+ }
+
/**
* Add the index row to unsafe.
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index a918611..fd78efc 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -35,25 +35,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.BLOCKLET_SIZE;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_DATA_FILE_VERSION;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_DATE_FORMAT;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_EXECUTOR_STARTUP_TIMEOUT;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.CSV_READ_BUFFER_SIZE;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_AUTO_HANDOFF;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_UNSAFE_SORT;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_VECTOR_READER;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.HANDOFF_SIZE;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.LOCK_TYPE;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.NUM_CORES;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.NUM_CORES_BLOCK_SORT;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.SORT_SIZE;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.*;
import static org.apache.carbondata.core.constants.CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB;
import static org.apache.carbondata.core.constants.CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO;
@@ -151,8 +133,8 @@ public final class CarbonProperties {
case HANDOFF_SIZE:
validateHandoffSize();
break;
- case CARBON_COMBINE_SMALL_INPUT_FILES:
- validateCombineSmallInputFiles();
+ case CARBON_TASK_DISTRIBUTION:
+ validateCarbonTaskDistribution();
break;
// The method validate the validity of configured carbon.timestamp.format value
// and reset to default value if validation fail
@@ -199,7 +181,7 @@ public final class CarbonProperties {
validateLockType();
validateCarbonCSVReadBufferSizeByte();
validateHandoffSize();
- validateCombineSmallInputFiles();
+ validateCarbonTaskDistribution();
// The method validate the validity of configured carbon.timestamp.format value
// and reset to default value if validation fail
validateTimeFormatKey(CARBON_TIMESTAMP_FORMAT,
@@ -361,22 +343,24 @@ public final class CarbonProperties {
if (!isValidBooleanValue) {
LOGGER.warn("The custom block distribution value \"" + customBlockDistributionStr
+ "\" is invalid. Using the default value \""
- + CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT);
- carbonProperties.setProperty(CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT);
+ + false);
+ carbonProperties.setProperty(CARBON_CUSTOM_BLOCK_DISTRIBUTION, "false");
}
}
- private void validateCombineSmallInputFiles() {
- String combineSmallInputFilesStr =
- carbonProperties.getProperty(CARBON_COMBINE_SMALL_INPUT_FILES);
- boolean isValidBooleanValue = CarbonUtil.validateBoolean(combineSmallInputFilesStr);
- if (!isValidBooleanValue) {
- LOGGER.warn("The combine small files value \"" + combineSmallInputFilesStr
+ private void validateCarbonTaskDistribution() {
+ String carbonTaskDistribution = carbonProperties.getProperty(CARBON_TASK_DISTRIBUTION);
+ boolean isValid = carbonTaskDistribution != null && (
+ carbonTaskDistribution.equalsIgnoreCase(CARBON_TASK_DISTRIBUTION_MERGE_FILES)
+ || carbonTaskDistribution.equalsIgnoreCase(CARBON_TASK_DISTRIBUTION_BLOCKLET)
+ || carbonTaskDistribution.equalsIgnoreCase(CARBON_TASK_DISTRIBUTION_BLOCK)
+ || carbonTaskDistribution.equalsIgnoreCase(CARBON_TASK_DISTRIBUTION_CUSTOM));
+ if (!isValid) {
+ LOGGER.warn("The carbon task distribution value \"" + carbonTaskDistribution
+ "\" is invalid. Using the default value \""
- + CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT);
- carbonProperties.setProperty(CARBON_COMBINE_SMALL_INPUT_FILES,
- CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT);
+ + CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT);
+ carbonProperties.setProperty(CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
index ae29b03..daf6db0 100644
--- a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java
@@ -77,7 +77,7 @@ public class CarbonPropertiesValidationTest extends TestCase {
String valueAfterValidation =
carbonProperties.getProperty(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION);
assertTrue(valueBeforeValidation.equals(valueAfterValidation));
- assertTrue(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT
+ assertTrue("false"
.equalsIgnoreCase(valueAfterValidation));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
index 15126b6..4104ea3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableOverwriteTestCase.scala
@@ -155,6 +155,25 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
checkAnswer(sql("select count(*) from weather6"), Seq(Row(2)))
}
+ test("Test overwrite static partition with wrong int value") {
+ sql(
+ """
+ | CREATE TABLE weather7 (type String)
+ | PARTITIONED BY (year int, month int, day int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql("insert into weather7 partition(year=2014, month=05, day=25) select 'rainy'")
+ sql("insert into weather7 partition(year=2014, month=04, day=23) select 'cloudy'")
+ sql("insert overwrite table weather7 partition(year=2014, month=05, day=25) select 'sunny'")
+ checkExistence(sql("select * from weather7"), true, "sunny")
+ checkAnswer(sql("select count(*) from weather7"), Seq(Row(2)))
+ sql("insert into weather7 partition(year=2014, month, day) select 'rainy1',06,25")
+ sql("insert into weather7 partition(year=2014, month=01, day) select 'rainy2',27")
+ sql("insert into weather7 partition(year=2014, month=01, day=02) select 'rainy3'")
+ checkAnswer(sql("select count(*) from weather7 where month=1"), Seq(Row(2)))
+ }
+
override def afterAll = {
dropTable
@@ -168,6 +187,7 @@ class StandardPartitionTableOverwriteTestCase extends QueryTest with BeforeAndAf
sql("drop table if exists insertstaticpartitiondynamic")
sql("drop table if exists partitionallcompaction")
sql("drop table if exists weather6")
+ sql("drop table if exists weather7")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index cc68b9c..a04e9e1 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -145,7 +145,9 @@ class CarbonScanRDD(
statistic.addStatistics(QueryStatisticsConstants.BLOCK_ALLOCATION, System.currentTimeMillis)
statisticRecorder.recordStatisticsForDriver(statistic, queryId)
statistic = new QueryStatistic()
-
+ val carbonDistribution = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT)
// If bucketing is enabled on table then partitions should be grouped based on buckets.
if (bucketedTable != null) {
var i = 0
@@ -161,101 +163,106 @@ class CarbonScanRDD(
i += 1
result.add(partition)
}
- } else if (CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT).toBoolean) {
- // create a list of block based on split
- val blockList = splits.asScala.map(_.asInstanceOf[Distributable])
-
- // get the list of executors and map blocks to executors based on locality
- val activeNodes = DistributionUtil.ensureExecutorsAndGetNodeList(blockList, sparkContext)
-
- // divide the blocks among the tasks of the nodes as per the data locality
- val nodeBlockMapping = CarbonLoaderUtil.nodeBlockTaskMapping(blockList.asJava, -1,
- parallelism, activeNodes.toList.asJava)
- var i = 0
- // Create Spark Partition for each task and assign blocks
- nodeBlockMapping.asScala.foreach { case (node, blockList) =>
- blockList.asScala.foreach { blocksPerTask =>
- val splits = blocksPerTask.asScala.map(_.asInstanceOf[CarbonInputSplit])
- if (blocksPerTask.size() != 0) {
- val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier, splits.asJava, Array(node))
- val partition = new CarbonSparkPartition(id, i, multiBlockSplit)
- result.add(partition)
- i += 1
+ } else {
+ val useCustomDistribution =
+ CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
+ "false").toBoolean ||
+ carbonDistribution.equalsIgnoreCase(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_CUSTOM)
+ if (useCustomDistribution) {
+ // create a list of block based on split
+ val blockList = splits.asScala.map(_.asInstanceOf[Distributable])
+
+ // get the list of executors and map blocks to executors based on locality
+ val activeNodes = DistributionUtil.ensureExecutorsAndGetNodeList(blockList, sparkContext)
+
+ // divide the blocks among the tasks of the nodes as per the data locality
+ val nodeBlockMapping = CarbonLoaderUtil.nodeBlockTaskMapping(blockList.asJava, -1,
+ parallelism, activeNodes.toList.asJava)
+ var i = 0
+ // Create Spark Partition for each task and assign blocks
+ nodeBlockMapping.asScala.foreach { case (node, blockList) =>
+ blockList.asScala.foreach { blocksPerTask =>
+ val splits = blocksPerTask.asScala.map(_.asInstanceOf[CarbonInputSplit])
+ if (blocksPerTask.size() != 0) {
+ val multiBlockSplit =
+ new CarbonMultiBlockSplit(identifier, splits.asJava, Array(node))
+ val partition = new CarbonSparkPartition(id, i, multiBlockSplit)
+ result.add(partition)
+ i += 1
+ }
}
}
- }
- noOfNodes = nodeBlockMapping.size
- } else if (CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_USE_BLOCKLET_DISTRIBUTION,
- CarbonCommonConstants.CARBON_USE_BLOCKLET_DISTRIBUTION_DEFAULT).toBoolean) {
- // Use blocklet distribution
- // Randomize the blocklets for better shuffling
- Random.shuffle(splits.asScala).zipWithIndex.foreach { splitWithIndex =>
- val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier,
- Seq(splitWithIndex._1.asInstanceOf[CarbonInputSplit]).asJava,
- splitWithIndex._1.getLocations)
- val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
- result.add(partition)
- }
- } else if (CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES,
- CarbonCommonConstants.CARBON_COMBINE_SMALL_INPUT_FILES_DEFAULT).toBoolean) {
-
- // sort blocks in reverse order of length
- val blockSplits = splits
- .asScala
- .map(_.asInstanceOf[CarbonInputSplit])
- .groupBy(f => f.getBlockPath)
- .map { blockSplitEntry =>
- new CarbonMultiBlockSplit(identifier,
- blockSplitEntry._2.asJava,
- blockSplitEntry._2.flatMap(f => f.getLocations).distinct.toArray)
- }.toArray.sortBy(_.getLength)(implicitly[Ordering[Long]].reverse)
-
- val defaultMaxSplitBytes = sessionState(spark).conf.filesMaxPartitionBytes
- val openCostInBytes = sessionState(spark).conf.filesOpenCostInBytes
- val defaultParallelism = spark.sparkContext.defaultParallelism
- val totalBytes = blockSplits.map(_.getLength + openCostInBytes).sum
- val bytesPerCore = totalBytes / defaultParallelism
-
- val maxSplitBytes = Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))
- LOGGER.info(s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " +
- s"open cost is considered as scanning $openCostInBytes bytes.")
-
- val currentFiles = new ArrayBuffer[CarbonMultiBlockSplit]
- var currentSize = 0L
-
- def closePartition(): Unit = {
- if (currentFiles.nonEmpty) {
- result.add(combineSplits(currentFiles, currentSize, result.size()))
+ noOfNodes = nodeBlockMapping.size
+ } else if (carbonDistribution.equalsIgnoreCase(
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_BLOCKLET)) {
+ // Use blocklet distribution
+ // Randomize the blocklets for better shuffling
+ Random.shuffle(splits.asScala).zipWithIndex.foreach { splitWithIndex =>
+ val multiBlockSplit =
+ new CarbonMultiBlockSplit(identifier,
+ Seq(splitWithIndex._1.asInstanceOf[CarbonInputSplit]).asJava,
+ splitWithIndex._1.getLocations)
+ val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
+ result.add(partition)
+ }
+ } else if (carbonDistribution.equalsIgnoreCase(
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_MERGE_FILES)) {
+
+ // sort blocks in reverse order of length
+ val blockSplits = splits
+ .asScala
+ .map(_.asInstanceOf[CarbonInputSplit])
+ .groupBy(f => f.getBlockPath)
+ .map { blockSplitEntry =>
+ new CarbonMultiBlockSplit(identifier,
+ blockSplitEntry._2.asJava,
+ blockSplitEntry._2.flatMap(f => f.getLocations).distinct.toArray)
+ }.toArray.sortBy(_.getLength)(implicitly[Ordering[Long]].reverse)
+
+ val defaultMaxSplitBytes = sessionState(spark).conf.filesMaxPartitionBytes
+ val openCostInBytes = sessionState(spark).conf.filesOpenCostInBytes
+ val defaultParallelism = spark.sparkContext.defaultParallelism
+ val totalBytes = blockSplits.map(_.getLength + openCostInBytes).sum
+ val bytesPerCore = totalBytes / defaultParallelism
+
+ val maxSplitBytes = Math
+ .min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))
+ LOGGER.info(s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " +
+ s"open cost is considered as scanning $openCostInBytes bytes.")
+
+ val currentFiles = new ArrayBuffer[CarbonMultiBlockSplit]
+ var currentSize = 0L
+
+ def closePartition(): Unit = {
+ if (currentFiles.nonEmpty) {
+ result.add(combineSplits(currentFiles, currentSize, result.size()))
+ }
+ currentFiles.clear()
+ currentSize = 0
}
- currentFiles.clear()
- currentSize = 0
- }
- blockSplits.foreach { file =>
- if (currentSize + file.getLength > maxSplitBytes) {
- closePartition()
+ blockSplits.foreach { file =>
+ if (currentSize + file.getLength > maxSplitBytes) {
+ closePartition()
+ }
+ // Add the given file to the current partition.
+ currentSize += file.getLength + openCostInBytes
+ currentFiles += file
+ }
+ closePartition()
+ } else {
+ // Use block distribution
+ splits.asScala.map(_.asInstanceOf[CarbonInputSplit]).groupBy { f =>
+ f.getSegmentId.concat(f.getBlockPath)
+ }.values.zipWithIndex.foreach { splitWithIndex =>
+ val multiBlockSplit =
+ new CarbonMultiBlockSplit(identifier,
+ splitWithIndex._1.asJava,
+ splitWithIndex._1.flatMap(f => f.getLocations).distinct.toArray)
+ val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
+ result.add(partition)
}
- // Add the given file to the current partition.
- currentSize += file.getLength + openCostInBytes
- currentFiles += file
- }
- closePartition()
- } else {
- // Use block distribution
- splits.asScala.map(_.asInstanceOf[CarbonInputSplit])
- .groupBy(f => f.getBlockPath).values.zipWithIndex.foreach { splitWithIndex =>
- val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier,
- splitWithIndex._1.asJava,
- splitWithIndex._1.flatMap(f => f.getLocations).distinct.toArray)
- val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
- result.add(partition)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 6c8a6b0..86d25b4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -177,9 +177,9 @@ object CarbonScalaUtil {
return null
}
dataType match {
- case TimestampType =>
+ case TimestampType if timeStampFormat != null =>
DateTimeUtils.timestampToString(timeStampFormat.parse(value).getTime * 1000)
- case DateType =>
+ case DateType if dateFormat != null =>
DateTimeUtils.dateToString(
(dateFormat.parse(value).getTime / DateTimeUtils.MILLIS_PER_DAY).toInt)
case ShortType => value.toShort.toString
@@ -233,16 +233,18 @@ object CarbonScalaUtil {
serializationNullFormat: String,
badRecordAction: String,
isEmptyBadRecord: Boolean): Map[String, String] = {
+ val hivedefaultpartition = "__HIVE_DEFAULT_PARTITION__"
partitionSpec.map{ case (col, pvalue) =>
// replace special string with empty value.
- val value = if (pvalue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ val value = if (pvalue == null) {
+ hivedefaultpartition
+ } else if (pvalue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
""
} else {
pvalue
}
val carbonColumn = table.getColumnByName(table.getTableName, col.toLowerCase)
val dataType = CarbonScalaUtil.convertCarbonToSparkDataType(carbonColumn.getDataType)
- val hivedefaultpartition = "__HIVE_DEFAULT_PARTITION__"
try {
if (isEmptyBadRecord && value.length == 0 &&
badRecordAction.equalsIgnoreCase(LoggerAction.IGNORE.toString) &&
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 9577615..6b43152 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -521,9 +521,10 @@ case class CarbonLoadDataCommand(
CarbonSession.threadSet(
CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
badRecordAction)
+ val isEmptyBadRecord = carbonLoadModel.getIsEmptyDataBadRecord.split(",")(1)
CarbonSession.threadSet(
CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
- carbonLoadModel.getIsEmptyDataBadRecord.split(",")(1))
+ isEmptyBadRecord)
try {
val query: LogicalPlan = if (dataFrame.isDefined) {
val delimiterLevel1 = carbonLoadModel.getComplexDelimiterLevel1
@@ -632,7 +633,13 @@ case class CarbonLoadDataCommand(
overwrite = false,
ifPartitionNotExists = false)
if (isOverwriteTable && partition.nonEmpty) {
- overwritePartition(sparkSession, table, convertedPlan)
+ overwritePartition(
+ sparkSession,
+ table,
+ convertedPlan,
+ serializationNullFormat,
+ badRecordAction,
+ isEmptyBadRecord.toBoolean)
} else {
Dataset.ofRows(sparkSession, convertedPlan)
}
@@ -754,11 +761,25 @@ case class CarbonLoadDataCommand(
private def overwritePartition(
sparkSession: SparkSession,
table: CarbonTable,
- logicalPlan: LogicalPlan): Unit = {
+ logicalPlan: LogicalPlan,
+ serializationNullFormat: String,
+ badRecordAction: String,
+ isEmptyBadRecord: Boolean): Unit = {
val identifier = TableIdentifier(table.getTableName, Some(table.getDatabaseName))
+
+ // Update the partitions as per the datatype expect for time and datetype as we
+ // expect user provides the format in standard spark/hive formats.
+ val updatedPartitions = CarbonScalaUtil.updatePartitions(
+ partition.filter(_._2.isDefined).map(f => (f._1, f._2.get)),
+ table,
+ timeFormat = null,
+ dateFormat = null,
+ serializationNullFormat,
+ badRecordAction,
+ isEmptyBadRecord)
val existingPartitions = sparkSession.sessionState.catalog.listPartitions(
identifier,
- Some(partition.filter(_._2.isDefined).map(f => (f._1, f._2.get))))
+ Some(updatedPartitions))
val partitionNames = existingPartitions.toList.flatMap { partition =>
partition.spec.seq.map{case (column, value) => column + "=" + value}
}.toSet
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
index 20a6bab..2f12bef 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
@@ -143,7 +143,7 @@ private[sql] case class CarbonProjectForUpdateCommand(
CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, e.compactionTimeStamp.toString)
case e: Exception =>
- LOGGER.error("Exception in update operation" + e)
+ LOGGER.error(e, "Exception in update operation")
// ****** start clean up.
// In case of failure , clean all related delete delta files
CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, currentTime + "")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index c43a204..d74e461 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -245,7 +245,7 @@ private class CarbonOutputWriter(path: String,
null
}
}
- lazy val partitionData = if (partitions.nonEmpty) {
+ lazy val (updatedPartitions, partitionData) = if (partitions.nonEmpty) {
val updatedPartitions = partitions.map{ p =>
val value = p.substring(p.indexOf("=") + 1, p.length)
val col = p.substring(0, p.indexOf("="))
@@ -273,24 +273,25 @@ private class CarbonOutputWriter(path: String,
dateFormatString = loadModel.getDefaultDateFormat
}
val dateFormat = new SimpleDateFormat(dateFormatString)
- updatedPartitions.map {case (col, value) =>
+ val formattedPartitions = updatedPartitions.map {case (col, value) =>
// Only convert the static partitions to the carbon format and use it while loading data
// to carbon.
if (staticPartition.getOrDefault(col, false)) {
- CarbonScalaUtil.convertToCarbonFormat(value,
+ (col, CarbonScalaUtil.convertToCarbonFormat(value,
CarbonScalaUtil.convertCarbonToSparkDataType(
table.getColumnByName(table.getTableName, col).getDataType),
timeFormat,
- dateFormat)
+ dateFormat))
} else {
- value
+ (col, value)
}
}
+ (formattedPartitions, formattedPartitions.map(_._2))
} else {
- updatedPartitions.map(_._2)
+ (updatedPartitions, updatedPartitions.map(_._2))
}
} else {
- Array.empty
+ (Map.empty[String, String].toArray, Array.empty)
}
val writable = new StringArrayWritable()
@@ -346,41 +347,17 @@ private class CarbonOutputWriter(path: String,
val isEmptyBadRecord = loadModel.getIsEmptyDataBadRecord.split(",")(1).toBoolean
// write partition info to new file.
val partitonList = new util.ArrayList[String]()
- val splitPartitions = partitions.map{ p =>
- val value = p.substring(p.indexOf("=") + 1, p.length)
- val col = p.substring(0, p.indexOf("="))
- (col, value)
- }.toMap
- val updatedPartitions =
- if (staticPartition != null) {
- // There can be scnerio like dynamic and static combination, in that case we should convert
- // only the dyanamic partition values to the proper format and store to carbon parttion map
- splitPartitions.map { case (col, value) =>
- if (!staticPartition.getOrDefault(col, false)) {
- CarbonScalaUtil.updatePartitions(
- Seq((col, value)).toMap,
- table,
- timeFormat,
- dateFormat,
- serializeFormat,
- badRecordAction,
- isEmptyBadRecord).toSeq.head
- } else {
- (col, value)
- }
- }
- } else {
- // All dynamic partitions need to be converted to proper format
- CarbonScalaUtil.updatePartitions(
- splitPartitions,
- table,
- timeFormat,
- dateFormat,
- serializeFormat,
- badRecordAction,
- isEmptyBadRecord)
- }
- updatedPartitions.foreach(p => partitonList.add(p._1 + "=" + p._2))
+ val formattedPartitions =
+ // All dynamic partitions need to be converted to proper format
+ CarbonScalaUtil.updatePartitions(
+ updatedPartitions.toMap,
+ table,
+ timeFormat,
+ dateFormat,
+ serializeFormat,
+ badRecordAction,
+ isEmptyBadRecord)
+ formattedPartitions.foreach(p => partitonList.add(p._1 + "=" + p._2))
new PartitionMapFileStore().writePartitionMapFile(
segmentPath,
loadModel.getTaskNo,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
index eb59184..93c7c09 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -182,10 +182,10 @@ object CarbonMetaStoreFactory {
def createCarbonMetaStore(conf: RuntimeConfig): CarbonMetaStore = {
val readSchemaFromHiveMetaStore = readSchemaFromHive(conf)
if (readSchemaFromHiveMetaStore) {
- LOGGER.info("Hive based carbon metastore is enabled")
+ LOGGER.audit("Hive based carbon metastore is enabled")
new CarbonHiveMetaStore()
} else {
- LOGGER.info("File based carbon metastore is enabled")
+ LOGGER.audit("File based carbon metastore is enabled")
new CarbonFileMetastore()
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala b/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
index 837b21f..15ccb0c 100644
--- a/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
+++ b/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
@@ -42,11 +42,11 @@ class CarbonSQLConf(sparkSession: SparkSession) {
CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
SQLConfigBuilder(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
- .doc("To enable/ disable carbon custom block distribution.")
- .booleanConf
+ .doc("To set carbon task distribution.")
+ .stringConf
.createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT).toBoolean)
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
val BAD_RECORDS_LOGGER_ENABLE =
SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
.doc("To enable/ disable carbon bad record logger.")
@@ -117,8 +117,8 @@ class CarbonSQLConf(sparkSession: SparkSession) {
CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT).toBoolean)
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4dc866f/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala b/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
index eef6604..2128ffd 100644
--- a/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
+++ b/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
@@ -41,11 +41,11 @@ class CarbonSQLConf(sparkSession: SparkSession) {
CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
buildConf(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
- .doc("To enable/ disable carbon custom block distribution.")
- .booleanConf
+ .doc("To set carbon task distribution.")
+ .stringConf
.createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT).toBoolean)
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
val BAD_RECORDS_LOGGER_ENABLE =
buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
.doc("To enable/ disable carbon bad record logger.")
@@ -116,8 +116,8 @@ class CarbonSQLConf(sparkSession: SparkSession) {
CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION_DEFAULT).toBoolean)
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
[28/50] [abbrv] carbondata git commit: [CI][Streaming] Reduce the
execution time of TestStreamingTableOperation test suite
Posted by ja...@apache.org.
[CI][Streaming] Reduce the execution time of TestStreamingTableOperation test suite
Combine test case to reduce the execution time of TestStreamingTableOperation test suite, but not reduce the test coverage.
This closes #1863
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5a82232a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5a82232a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5a82232a
Branch: refs/heads/carbonstore
Commit: 5a82232a85abf97bc6fad7c8b0ab036e5d74eab2
Parents: 2f4dbb6
Author: QiangCai <qi...@qq.com>
Authored: Fri Jan 26 09:40:46 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Sun Jan 28 13:05:32 2018 +0530
----------------------------------------------------------------------
.../TestStreamingTableOperation.scala | 659 +++++--------------
1 file changed, 170 insertions(+), 489 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5a82232a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 62076bf..3de1391 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -63,42 +63,23 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
createTable(tableName = "batch_table", streaming = false, withBatchLoad = true)
// 2. streaming table with different input source
- // socket source
- createTable(tableName = "stream_table_socket", streaming = true, withBatchLoad = true)
// file source
createTable(tableName = "stream_table_file", streaming = true, withBatchLoad = true)
// 3. streaming table with bad records
- createTable(tableName = "bad_record_force", streaming = true, withBatchLoad = true)
createTable(tableName = "bad_record_fail", streaming = true, withBatchLoad = true)
// 4. streaming frequency check
createTable(tableName = "stream_table_1s", streaming = true, withBatchLoad = true)
- createTable(tableName = "stream_table_10s", streaming = true, withBatchLoad = true)
// 5. streaming table execute batch loading
- createTable(tableName = "stream_table_batch", streaming = true, withBatchLoad = false)
-
// 6. detail query
- // full scan
- createTable(tableName = "stream_table_scan", streaming = true, withBatchLoad = true)
- createTableWithComplexType(
- tableName = "stream_table_scan_complex", streaming = true, withBatchLoad = true)
- // filter scan
+ // 8. compaction
+ // full scan + filter scan + aggregate query
createTable(tableName = "stream_table_filter", streaming = true, withBatchLoad = true)
- createTableWithComplexType(
- tableName = "stream_table_filter_complex", streaming = true, withBatchLoad = true)
- // 7. aggregate query
- createTable(tableName = "stream_table_agg", streaming = true, withBatchLoad = true)
createTableWithComplexType(
- tableName = "stream_table_agg_complex", streaming = true, withBatchLoad = true)
-
- // 8. compaction
- createTable(tableName = "stream_table_compact", streaming = true, withBatchLoad = true)
-
- // 9. create new stream segment if current stream segment is full
- createTable(tableName = "stream_table_new", streaming = true, withBatchLoad = true)
+ tableName = "stream_table_filter_complex", streaming = true, withBatchLoad = true)
// 10. fault tolerant
createTable(tableName = "stream_table_tolerant", streaming = true, withBatchLoad = true)
@@ -108,22 +89,13 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
createTable(tableName = "stream_table_delete_date", streaming = true, withBatchLoad = false)
// 12. reject alter streaming properties
- createTable(tableName = "stream_table_alter", streaming = false, withBatchLoad = false)
-
- // 13. handoff streaming segment
- createTable(tableName = "stream_table_handoff", streaming = true, withBatchLoad = false)
-
- // 14. finish streaming
- createTable(tableName = "stream_table_finish", streaming = true, withBatchLoad = true)
+ // 13. handoff streaming segment and finish streaming
+ createTable(tableName = "stream_table_handoff", streaming = false, withBatchLoad = false)
// 15. auto handoff streaming segment
- createTable(tableName = "stream_table_auto_handoff", streaming = true, withBatchLoad = false)
-
// 16. close streaming table
- createTable(tableName = "stream_table_close", streaming = true, withBatchLoad = false)
- createTable(tableName = "stream_table_close_auto_handoff", streaming = true, withBatchLoad = false)
-
// 17. reopen streaming table after close
+ // 9. create new stream segment if current stream segment is full
createTable(tableName = "stream_table_reopen", streaming = true, withBatchLoad = false)
// 18. block drop table while streaming is in progress
@@ -193,30 +165,15 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
def dropTable(): Unit = {
sql("drop table if exists streaming.batch_table")
- sql("drop table if exists streaming.stream_table_socket")
sql("drop table if exists streaming.stream_table_file")
- sql("drop table if exists streaming.bad_record_force")
sql("drop table if exists streaming.bad_record_fail")
sql("drop table if exists streaming.stream_table_1s")
- sql("drop table if exists streaming.stream_table_10s")
- sql("drop table if exists streaming.stream_table_batch")
- sql("drop table if exists streaming.stream_table_scan")
- sql("drop table if exists streaming.stream_table_scan_complex")
- sql("drop table if exists streaming.stream_table_filter")
+ sql("drop table if exists streaming.stream_table_filter ")
sql("drop table if exists streaming.stream_table_filter_complex")
- sql("drop table if exists streaming.stream_table_agg")
- sql("drop table if exists streaming.stream_table_agg_complex")
- sql("drop table if exists streaming.stream_table_compact")
- sql("drop table if exists streaming.stream_table_new")
sql("drop table if exists streaming.stream_table_tolerant")
sql("drop table if exists streaming.stream_table_delete_id")
sql("drop table if exists streaming.stream_table_delete_date")
- sql("drop table if exists streaming.stream_table_alter")
sql("drop table if exists streaming.stream_table_handoff")
- sql("drop table if exists streaming.stream_table_finish")
- sql("drop table if exists streaming.stream_table_auto_handoff")
- sql("drop table if exists streaming.stream_table_close")
- sql("drop table if exists streaming.stream_table_close_auto_handoff")
sql("drop table if exists streaming.stream_table_reopen")
sql("drop table if exists streaming.stream_table_drop")
sql("drop table if exists streaming.agg_table_block")
@@ -254,26 +211,6 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
}
}
- // input source: socket
- test("streaming ingest from socket source") {
- executeStreamingIngest(
- tableName = "stream_table_socket",
- batchNums = 2,
- rowNumsEachBatch = 10,
- intervalOfSource = 1,
- intervalOfIngest = 1,
- continueSeconds = 10,
- generateBadRecords = false,
- badRecordAction = "force",
- autoHandoff = false
- )
-
- checkAnswer(
- sql("select count(*) from streaming.stream_table_socket"),
- Seq(Row(25))
- )
- }
-
// input source: file
test("streaming ingest from file source") {
val identifier = new TableIdentifier("stream_table_file", Option("streaming"))
@@ -288,7 +225,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
thread.start()
Thread.sleep(2000)
generateCSVDataFile(spark, idStart = 30, rowNums = 10, csvDataDir)
- Thread.sleep(10000)
+ Thread.sleep(5000)
thread.interrupt()
checkAnswer(
sql("select count(*) from streaming.stream_table_file"),
@@ -297,25 +234,6 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
}
// bad records
- test("streaming table with bad records action: force") {
- executeStreamingIngest(
- tableName = "bad_record_force",
- batchNums = 2,
- rowNumsEachBatch = 10,
- intervalOfSource = 1,
- intervalOfIngest = 1,
- continueSeconds = 10,
- generateBadRecords = true,
- badRecordAction = "force",
- autoHandoff = false
- )
- checkAnswer(
- sql("select count(*) from streaming.stream_table_socket"),
- Seq(Row(25))
- )
-
- }
-
test("streaming table with bad records action: fail") {
executeStreamingIngest(
tableName = "bad_record_fail",
@@ -323,138 +241,58 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
rowNumsEachBatch = 10,
intervalOfSource = 1,
intervalOfIngest = 1,
- continueSeconds = 10,
+ continueSeconds = 8,
generateBadRecords = true,
badRecordAction = "fail",
autoHandoff = false
)
val result = sql("select count(*) from streaming.bad_record_fail").collect()
- assert(result(0).getLong(0) < 25)
+ assert(result(0).getLong(0) < 10 + 5)
}
// ingest with different interval
test("1 row per 1 second interval") {
executeStreamingIngest(
tableName = "stream_table_1s",
- batchNums = 20,
+ batchNums = 3,
rowNumsEachBatch = 1,
intervalOfSource = 1,
intervalOfIngest = 1,
- continueSeconds = 20,
+ continueSeconds = 6,
generateBadRecords = false,
badRecordAction = "force",
autoHandoff = false
)
val result = sql("select count(*) from streaming.stream_table_1s").collect()
// 20 seconds can't ingest all data, exists data delay
- assert(result(0).getLong(0) > 5 + 10)
+ assert(result(0).getLong(0) > 5)
}
- test("10000 row per 10 seconds interval") {
+ test("query on stream table with dictionary, sort_columns") {
executeStreamingIngest(
- tableName = "stream_table_10s",
- batchNums = 5,
- rowNumsEachBatch = 10000,
+ tableName = "stream_table_filter",
+ batchNums = 2,
+ rowNumsEachBatch = 25,
intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 50,
- generateBadRecords = false,
- badRecordAction = "force",
- autoHandoff = false
- )
- checkAnswer(
- sql("select count(*) from streaming.stream_table_10s"),
- Seq(Row(5 + 10000 * 5)))
- }
-
- // batch loading on streaming table
- test("streaming table execute batch loading") {
- executeStreamingIngest(
- tableName = "stream_table_batch",
- batchNums = 5,
- rowNumsEachBatch = 100,
- intervalOfSource = 3,
intervalOfIngest = 5,
- continueSeconds = 30,
- generateBadRecords = false,
- badRecordAction = "force",
- autoHandoff = false
- )
- checkAnswer(
- sql("select count(*) from streaming.stream_table_batch"),
- Seq(Row(100 * 5)))
-
- executeBatchLoad("stream_table_batch")
-
- checkAnswer(
- sql("select count(*) from streaming.stream_table_batch"),
- Seq(Row(100 * 5 + 5)))
- }
-
- // detail query on batch and stream segment
- test("non-filter query on stream table with dictionary, sort_columns") {
- executeStreamingIngest(
- tableName = "stream_table_scan",
- batchNums = 5,
- rowNumsEachBatch = 10,
- intervalOfSource = 2,
- intervalOfIngest = 4,
- continueSeconds = 20,
- generateBadRecords = false,
- badRecordAction = "force",
- autoHandoff = false
- )
-
- val result = sql("select * from streaming.stream_table_scan order by id").collect()
- assert(result != null)
- assert(result.length == 55)
- // check one row of streaming data
- assert(result(0).getInt(0) == 1)
- assert(result(0).getString(1) == "name_1")
- // check one row of batch loading
- assert(result(50).getInt(0) == 100000001)
- assert(result(50).getString(1) == "batch_1")
- }
-
- test("non-filter query on stream table with dictionary, sort_columns and complex column") {
- executeStreamingIngest(
- tableName = "stream_table_scan_complex",
- batchNums = 5,
- rowNumsEachBatch = 10,
- intervalOfSource = 2,
- intervalOfIngest = 4,
continueSeconds = 20,
- generateBadRecords = false,
+ generateBadRecords = true,
badRecordAction = "force",
autoHandoff = false
)
- val result = sql("select * from streaming.stream_table_scan_complex order by id").collect()
+ // non-filter
+ val result = sql("select * from streaming.stream_table_filter order by id").collect()
assert(result != null)
assert(result.length == 55)
// check one row of streaming data
- assert(result(0).getInt(0) == 1)
- assert(result(0).getString(1) == "name_1")
- assert(result(0).getStruct(4).getInt(1) == 1)
+ assert(result(1).getInt(0) == 1)
+ assert(result(1).getString(1) == "name_1")
// check one row of batch loading
assert(result(50).getInt(0) == 100000001)
assert(result(50).getString(1) == "batch_1")
- assert(result(50).getStruct(4).getInt(1) == 20)
- }
-
- test("filter query on stream table with dictionary, sort_columns") {
- executeStreamingIngest(
- tableName = "stream_table_filter",
- batchNums = 5,
- rowNumsEachBatch = 10,
- intervalOfSource = 2,
- intervalOfIngest = 4,
- continueSeconds = 20,
- generateBadRecords = true,
- badRecordAction = "force",
- autoHandoff = false
- )
+ // filter
checkAnswer(
sql("select * from stream_table_filter where id = 1"),
Seq(Row(1, "name_1", "city_1", 10000.0)))
@@ -481,21 +319,77 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
sql("select * from stream_table_filter where city = ''"),
Seq(Row(2, "name_2", "", 20000.0)))
+ // agg
+ checkAnswer(
+ sql("select count(*), max(id), min(name), cast(avg(id) as integer), sum(id) " +
+ "from stream_table_filter where id >= 2 and id <= 100000004"),
+ Seq(Row(52, 100000004, "batch_1", 7692332, 400001278)))
+
+ checkAnswer(
+ sql("select city, count(id), sum(id), cast(avg(id) as integer), " +
+ "max(salary), min(salary) " +
+ "from stream_table_filter " +
+ "where name in ('batch_1', 'batch_2', 'batch_3', 'name_1', 'name_2', 'name_3') " +
+ "and city <> '' " +
+ "group by city " +
+ "order by city"),
+ Seq(Row("city_1", 2, 100000002, 50000001, 10000.0, 0.1),
+ Row("city_2", 1, 100000002, 100000002, 0.2, 0.2),
+ Row("city_3", 2, 100000006, 50000003, 30000.0, 0.3)))
+
+ // batch loading
+ for(_ <- 0 to 2) {
+ executeBatchLoad("stream_table_filter")
+ }
+ checkAnswer(
+ sql("select count(*) from streaming.stream_table_filter"),
+ Seq(Row(25 * 2 + 5 + 5 * 3)))
+
+ sql("alter table streaming.stream_table_filter compact 'minor'")
+ Thread.sleep(5000)
+ val result1 = sql("show segments for table streaming.stream_table_filter").collect()
+ result1.foreach { row =>
+ if (row.getString(0).equals("1")) {
+ assertResult(SegmentStatus.STREAMING.getMessage)(row.getString(1))
+ assertResult(FileFormat.ROW_V1.toString)(row.getString(5))
+ } else if (row.getString(0).equals("0.1")) {
+ assertResult(SegmentStatus.SUCCESS.getMessage)(row.getString(1))
+ assertResult(FileFormat.COLUMNAR_V3.toString)(row.getString(5))
+ } else {
+ assertResult(SegmentStatus.COMPACTED.getMessage)(row.getString(1))
+ assertResult(FileFormat.COLUMNAR_V3.toString)(row.getString(5))
+ }
+ }
+
}
- test("filter query on stream table with dictionary, sort_columns and complex column") {
+ test("query on stream table with dictionary, sort_columns and complex column") {
executeStreamingIngest(
tableName = "stream_table_filter_complex",
- batchNums = 5,
- rowNumsEachBatch = 10,
- intervalOfSource = 2,
- intervalOfIngest = 4,
+ batchNums = 2,
+ rowNumsEachBatch = 25,
+ intervalOfSource = 5,
+ intervalOfIngest = 5,
continueSeconds = 20,
generateBadRecords = true,
badRecordAction = "force",
autoHandoff = false
)
+ // non-filter
+ val result = sql("select * from streaming.stream_table_filter_complex order by id").collect()
+ assert(result != null)
+ assert(result.length == 55)
+ // check one row of streaming data
+ assert(result(0).isNullAt(0))
+ assert(result(0).getString(1) == "name_6")
+ assert(result(0).getStruct(4).getInt(1) == 6)
+ // check one row of batch loading
+ assert(result(50).getInt(0) == 100000001)
+ assert(result(50).getString(1) == "batch_1")
+ assert(result(50).getStruct(4).getInt(1) == 20)
+
+ // filter
checkAnswer(
sql("select * from stream_table_filter_complex where id = 1"),
Seq(Row(1, "name_1", "city_1", 10000.0, Row(wrap(Array("school_1", "school_11")), 1))))
@@ -526,62 +420,16 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
sql("select * from stream_table_filter_complex where city = ''"),
Seq(Row(2, "name_2", "", 20000.0, Row(wrap(Array("school_2", "school_22")), 2))))
- }
-
- // aggregation
- test("aggregation query") {
- executeStreamingIngest(
- tableName = "stream_table_agg",
- batchNums = 5,
- rowNumsEachBatch = 10,
- intervalOfSource = 2,
- intervalOfIngest = 4,
- continueSeconds = 20,
- generateBadRecords = true,
- badRecordAction = "force",
- autoHandoff = false
- )
-
- checkAnswer(
- sql("select count(*), max(id), min(name), cast(avg(id) as integer), sum(id) " +
- "from stream_table_agg where id >= 2 and id <= 100000004"),
- Seq(Row(52, 100000004, "batch_1", 7692332, 400001278)))
-
- checkAnswer(
- sql("select city, count(id), sum(id), cast(avg(id) as integer), " +
- "max(salary), min(salary) " +
- "from stream_table_agg " +
- "where name in ('batch_1', 'batch_2', 'batch_3', 'name_1', 'name_2', 'name_3') " +
- "and city <> '' " +
- "group by city " +
- "order by city"),
- Seq(Row("city_1", 2, 100000002, 50000001, 10000.0, 0.1),
- Row("city_2", 1, 100000002, 100000002, 0.2, 0.2),
- Row("city_3", 2, 100000006, 50000003, 30000.0, 0.3)))
- }
-
- test("aggregation query with complex") {
- executeStreamingIngest(
- tableName = "stream_table_agg_complex",
- batchNums = 5,
- rowNumsEachBatch = 10,
- intervalOfSource = 2,
- intervalOfIngest = 4,
- continueSeconds = 20,
- generateBadRecords = true,
- badRecordAction = "force",
- autoHandoff = false
- )
-
+ // agg
checkAnswer(
sql("select count(*), max(id), min(name), cast(avg(file.age) as integer), sum(file.age) " +
- "from stream_table_agg_complex where id >= 2 and id <= 100000004"),
+ "from stream_table_filter_complex where id >= 2 and id <= 100000004"),
Seq(Row(52, 100000004, "batch_1", 27, 1408)))
checkAnswer(
sql("select city, count(id), sum(id), cast(avg(file.age) as integer), " +
"max(salary), min(salary) " +
- "from stream_table_agg_complex " +
+ "from stream_table_filter_complex " +
"where name in ('batch_1', 'batch_2', 'batch_3', 'name_1', 'name_2', 'name_3') " +
"and city <> '' " +
"group by city " +
@@ -591,70 +439,19 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
Row("city_3", 2, 100000006, 21, 30000.0, 0.3)))
}
- // compaction
- test("test compaction on stream table") {
- executeStreamingIngest(
- tableName = "stream_table_compact",
- batchNums = 5,
- rowNumsEachBatch = 10,
- intervalOfSource = 2,
- intervalOfIngest = 4,
- continueSeconds = 20,
- generateBadRecords = false,
- badRecordAction = "force",
- autoHandoff = false
- )
- for (_ <- 0 to 3) {
- executeBatchLoad("stream_table_compact")
- }
-
- sql("alter table streaming.stream_table_compact compact 'minor'")
-
- val result = sql("show segments for table streaming.stream_table_compact").collect()
- result.foreach { row =>
- if (row.getString(0).equals("1")) {
- assertResult(SegmentStatus.STREAMING.getMessage)(row.getString(1))
- assertResult(FileFormat.ROW_V1.toString)(row.getString(5))
- }
- }
- }
-
- // stream segment max size
- test("create new stream segment if current stream segment is full") {
- executeStreamingIngest(
- tableName = "stream_table_new",
- batchNums = 6,
- rowNumsEachBatch = 10000,
- intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 40,
- generateBadRecords = false,
- badRecordAction = "force",
- handoffSize = 1024L * 200,
- autoHandoff = false
- )
- assert(sql("show segments for table streaming.stream_table_new").count() > 1)
-
- checkAnswer(
- sql("select count(*) from streaming.stream_table_new"),
- Seq(Row(5 + 10000 * 6))
- )
- }
-
test("test deleting streaming segment by ID while ingesting") {
executeStreamingIngest(
tableName = "stream_table_delete_id",
- batchNums = 6,
- rowNumsEachBatch = 10000,
- intervalOfSource = 3,
+ batchNums = 3,
+ rowNumsEachBatch = 100,
+ intervalOfSource = 5,
intervalOfIngest = 5,
- continueSeconds = 20,
+ continueSeconds = 18,
generateBadRecords = false,
badRecordAction = "force",
- handoffSize = 1024L * 200,
+ handoffSize = 1,
autoHandoff = false
)
- Thread.sleep(10000)
val beforeDelete = sql("show segments for table streaming.stream_table_delete_id").collect()
val segmentIds1 = beforeDelete.filter(_.getString(1).equals("Streaming")).map(_.getString(0)).mkString(",")
val msg = intercept[Exception] {
@@ -674,17 +471,16 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
test("test deleting streaming segment by date while ingesting") {
executeStreamingIngest(
tableName = "stream_table_delete_date",
- batchNums = 6,
- rowNumsEachBatch = 10000,
- intervalOfSource = 3,
+ batchNums = 3,
+ rowNumsEachBatch = 100,
+ intervalOfSource = 5,
intervalOfIngest = 5,
- continueSeconds = 20,
+ continueSeconds = 18,
generateBadRecords = false,
badRecordAction = "force",
- handoffSize = 1024L * 200,
+ handoffSize = 1,
autoHandoff = false
)
- Thread.sleep(10000)
val beforeDelete = sql("show segments for table streaming.stream_table_delete_date").collect()
sql(s"delete from table streaming.stream_table_delete_date where segment.starttime before " +
s"'2999-10-01 01:00:00'")
@@ -696,230 +492,99 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
}
}
- test("reject alter streaming properties") {
+ test("reject alter streaming properties and handoff 'streaming finish' segment to columnar segment") {
try {
- sql("ALTER TABLE streaming.stream_table_alter UNSET TBLPROPERTIES IF EXISTS ('streaming')")
+ sql("ALTER TABLE streaming.stream_table_handoff UNSET TBLPROPERTIES IF EXISTS ('streaming')")
assert(false, "unsupport to unset streaming property")
} catch {
case _ =>
assert(true)
}
try {
- sql("ALTER TABLE streaming.stream_table_alter SET TBLPROPERTIES('streaming'='true')")
+ sql("ALTER TABLE streaming.stream_table_handoff SET TBLPROPERTIES('streaming'='true')")
executeStreamingIngest(
- tableName = "stream_table_alter",
- batchNums = 6,
- rowNumsEachBatch = 10000,
+ tableName = "stream_table_handoff",
+ batchNums = 2,
+ rowNumsEachBatch = 100,
intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 40,
+ intervalOfIngest = 5,
+ continueSeconds = 20,
generateBadRecords = false,
badRecordAction = "force",
- handoffSize = 1024L * 200,
+ handoffSize = 1L,
autoHandoff = false
)
- checkAnswer(
- sql("select count(*) from streaming.stream_table_alter"),
- Seq(Row(6 * 10000))
- )
} catch {
case _ =>
assert(false, "should support set table to streaming")
}
-
- try {
- sql("ALTER TABLE stream_table_alter SET TBLPROPERTIES('streaming'='false')")
- assert(false, "unsupport disable streaming properties")
- } catch {
- case _ =>
- assert(true)
- }
- }
-
- test("handoff 'streaming finish' segment to columnar segment") {
- executeStreamingIngest(
- tableName = "stream_table_handoff",
- batchNums = 6,
- rowNumsEachBatch = 10000,
- intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 40,
- generateBadRecords = false,
- badRecordAction = "force",
- handoffSize = 1024L * 200,
- autoHandoff = false
- )
val segments = sql("show segments for table streaming.stream_table_handoff").collect()
- assert(segments.length == 3 || segments.length == 4)
+ assert(segments.length == 2 || segments.length == 3)
assertResult("Streaming")(segments(0).getString(1))
- (1 to segments.length - 1).foreach { index =>
- assertResult("Streaming Finish")(segments(index).getString(1))
- }
+ assertResult("Streaming Finish")(segments(1).getString(1))
checkAnswer(
sql("select count(*) from streaming.stream_table_handoff"),
- Seq(Row(6 * 10000))
+ Seq(Row(2 * 100))
)
sql("alter table streaming.stream_table_handoff compact 'streaming'")
- Thread.sleep(10000)
+ Thread.sleep(5000)
val newSegments = sql("show segments for table streaming.stream_table_handoff").collect()
- assertResult(5)(newSegments.length)
- assertResult("Success")(newSegments(0).getString(1))
- assertResult("Success")(newSegments(1).getString(1))
- assertResult("Streaming")(newSegments(2).getString(1))
- assertResult("Compacted")(newSegments(3).getString(1))
- assertResult("Compacted")(newSegments(4).getString(1))
- checkAnswer(
- sql("select count(*) from streaming.stream_table_handoff"),
- Seq(Row(6 * 10000))
- )
- }
-
- test("auto handoff 'streaming finish' segment to columnar segment") {
- executeStreamingIngest(
- tableName = "stream_table_auto_handoff",
- batchNums = 6,
- rowNumsEachBatch = 10000,
- intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 40,
- generateBadRecords = false,
- badRecordAction = "force",
- handoffSize = 1024L * 200,
- autoHandoff = true
- )
- Thread.sleep(10000)
- val segments = sql("show segments for table streaming.stream_table_auto_handoff").collect()
- assertResult(5)(segments.length)
- assertResult(2)(segments.filter(_.getString(1).equals("Success")).length)
- assertResult(2)(segments.filter(_.getString(1).equals("Compacted")).length)
- assertResult(1)(segments.filter(_.getString(1).equals("Streaming")).length)
- checkAnswer(
- sql("select count(*) from streaming.stream_table_auto_handoff"),
- Seq(Row(6 * 10000))
- )
- }
-
- test("alter table finish streaming") {
- executeStreamingIngest(
- tableName = "stream_table_finish",
- batchNums = 6,
- rowNumsEachBatch = 10000,
- intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 40,
- generateBadRecords = false,
- badRecordAction = "force",
- handoffSize = 1024L * 200,
- autoHandoff = false
- )
- sql("alter table streaming.stream_table_finish finish streaming")
-
- val segments = sql("show segments for table streaming.stream_table_finish").collect()
- assert(segments.length == 4 || segments.length == 5)
- (0 to segments.length -2).foreach { index =>
- assertResult("Streaming Finish")(segments(index).getString(1))
+ assert(newSegments.length == 3 || newSegments.length == 5)
+ assertResult("Streaming")(newSegments((newSegments.length - 1) / 2).getString(1))
+ (0 until (newSegments.length - 1) / 2).foreach{ i =>
+ assertResult("Success")(newSegments(i).getString(1))
+ }
+ ((newSegments.length + 1) / 2 until newSegments.length).foreach{ i =>
+ assertResult("Compacted")(newSegments(i).getString(1))
}
- assertResult("Success")(segments(segments.length - 1).getString(1))
- checkAnswer(
- sql("select count(*) from streaming.stream_table_finish"),
- Seq(Row(5 + 6 * 10000))
- )
- }
-
- test("alter table close streaming") {
- executeStreamingIngest(
- tableName = "stream_table_close",
- batchNums = 6,
- rowNumsEachBatch = 10000,
- intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 40,
- generateBadRecords = false,
- badRecordAction = "force",
- handoffSize = 1024L * 200,
- autoHandoff = false
- )
-
- val table1 = CarbonEnv.getCarbonTable(Option("streaming"), "stream_table_close")(spark)
- assertResult(true)(table1.isStreamingTable)
- sql("alter table streaming.stream_table_close compact 'close_streaming'")
-
- val segments = sql("show segments for table streaming.stream_table_close").collect()
- assertResult(6)(segments.length)
- assertResult(3)(segments.filter(_.getString(1).equals("Success")).length)
- assertResult(3)(segments.filter(_.getString(1).equals("Compacted")).length)
- checkAnswer(
- sql("select count(*) from streaming.stream_table_close"),
- Seq(Row(6 * 10000))
- )
- val table2 = CarbonEnv.getCarbonTable(Option("streaming"), "stream_table_close")(spark)
- assertResult(false)(table2.isStreamingTable)
- }
-
- test("alter table close streaming with auto handoff") {
- executeStreamingIngest(
- tableName = "stream_table_close_auto_handoff",
- batchNums = 6,
- rowNumsEachBatch = 10000,
- intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 40,
- generateBadRecords = false,
- badRecordAction = "force",
- handoffSize = 1024L * 200,
- autoHandoff = true
- )
- Thread.sleep(10000)
- val table1 =
- CarbonEnv.getCarbonTable(Option("streaming"), "stream_table_close_auto_handoff")(spark)
- assertResult(true)(table1.isStreamingTable)
+ sql("alter table streaming.stream_table_handoff finish streaming")
+ val newSegments1 = sql("show segments for table streaming.stream_table_handoff").collect()
+ assertResult("Streaming Finish")(newSegments1((newSegments.length - 1) / 2).getString(1))
- sql("alter table streaming.stream_table_close_auto_handoff compact 'close_streaming'")
- val segments =
- sql("show segments for table streaming.stream_table_close_auto_handoff").collect()
- assertResult(6)(segments.length)
- assertResult(3)(segments.filter(_.getString(1).equals("Success")).length)
- assertResult(3)(segments.filter(_.getString(1).equals("Compacted")).length)
checkAnswer(
- sql("select count(*) from streaming.stream_table_close_auto_handoff"),
- Seq(Row(6 * 10000))
+ sql("select count(*) from streaming.stream_table_handoff"),
+ Seq(Row(2 * 100))
)
- val table2 =
- CarbonEnv.getCarbonTable(Option("streaming"), "stream_table_close_auto_handoff")(spark)
- assertResult(false)(table2.isStreamingTable)
+ try {
+ sql("ALTER TABLE stream_table_handoff SET TBLPROPERTIES('streaming'='false')")
+ assert(false, "unsupport disable streaming properties")
+ } catch {
+ case _ =>
+ assert(true)
+ }
}
- test("reopen streaming table") {
+ test("auto hand off, close and reopen streaming table") {
executeStreamingIngest(
tableName = "stream_table_reopen",
- batchNums = 6,
- rowNumsEachBatch = 10000,
+ batchNums = 2,
+ rowNumsEachBatch = 100,
intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 40,
+ intervalOfIngest = 5,
+ continueSeconds = 20,
generateBadRecords = false,
badRecordAction = "force",
- handoffSize = 1024L * 200,
- autoHandoff = true
+ handoffSize = 1L,
+ autoHandoff = false
)
- Thread.sleep(10000)
-
val table1 =
CarbonEnv.getCarbonTable(Option("streaming"), "stream_table_reopen")(spark)
assertResult(true)(table1.isStreamingTable)
sql("alter table streaming.stream_table_reopen compact 'close_streaming'")
+
val segments =
sql("show segments for table streaming.stream_table_reopen").collect()
- assertResult(6)(segments.length)
- assertResult(3)(segments.filter(_.getString(1).equals("Success")).length)
- assertResult(3)(segments.filter(_.getString(1).equals("Compacted")).length)
+ assert(segments.length == 4 || segments.length == 6)
+ assertResult(segments.length / 2)(segments.filter(_.getString(1).equals("Success")).length)
+ assertResult(segments.length / 2)(segments.filter(_.getString(1).equals("Compacted")).length)
+
checkAnswer(
sql("select count(*) from streaming.stream_table_reopen"),
- Seq(Row(6 * 10000))
+ Seq(Row(2 * 100))
)
val table2 =
@@ -934,21 +599,34 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
executeStreamingIngest(
tableName = "stream_table_reopen",
- batchNums = 6,
- rowNumsEachBatch = 10000,
+ batchNums = 2,
+ rowNumsEachBatch = 100,
intervalOfSource = 5,
- intervalOfIngest = 10,
- continueSeconds = 40,
+ intervalOfIngest = 5,
+ continueSeconds = 20,
generateBadRecords = false,
badRecordAction = "force",
- handoffSize = 1024L * 200,
+ handoffSize = 1L,
autoHandoff = true
)
Thread.sleep(10000)
+ val newSegments1 =
+ sql("show segments for table streaming.stream_table_reopen").collect()
+ assert(newSegments1.length == 7 || newSegments1.length == 9 || newSegments1.length == 11)
+ assertResult(1)(newSegments1.filter(_.getString(1).equals("Streaming")).length)
+ assertResult((newSegments1.length - 1) / 2)(newSegments1.filter(_.getString(1).equals("Success")).length)
+ assertResult((newSegments1.length - 1) / 2)(newSegments1.filter(_.getString(1).equals("Compacted")).length)
+
+ sql("alter table streaming.stream_table_reopen compact 'close_streaming'")
+ val newSegments =
+ sql("show segments for table streaming.stream_table_reopen").collect()
+ assert(newSegments.length == 8 || newSegments.length == 10 || newSegments.length == 12)
+ assertResult(newSegments.length / 2)(newSegments.filter(_.getString(1).equals("Success")).length)
+ assertResult(newSegments.length / 2)(newSegments.filter(_.getString(1).equals("Compacted")).length)
checkAnswer(
sql("select count(*) from streaming.stream_table_reopen"),
- Seq(Row(6 * 10000 * 2))
+ Seq(Row(2 * 100 * 2))
)
}
@@ -960,7 +638,7 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
var server: ServerSocket = null
try {
server = getServerSocket
- val thread1 = createWriteSocketThread(server, 2, 10, 5)
+ val thread1 = createWriteSocketThread(server, 2, 10, 3)
val thread2 = createSocketStreamingThread(spark, server.getLocalPort, tablePath, identifier, "force", 5, 1024L * 200, false)
thread1.start()
thread2.start()
@@ -1021,30 +699,33 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
var index = 0
for (_ <- 1 to writeNums) {
// write 5 records per iteration
+ val stringBuilder = new StringBuilder()
for (_ <- 1 to rowNums) {
index = index + 1
if (badRecords) {
if (index == 2) {
// null value
- socketWriter.println(index.toString + ",name_" + index
+ stringBuilder.append(index.toString + ",name_" + index
+ ",," + (10000.00 * index).toString +
",school_" + index + ":school_" + index + index + "$" + index)
} else if (index == 6) {
// illegal number
- socketWriter.println(index.toString + "abc,name_" + index
+ stringBuilder.append(index.toString + "abc,name_" + index
+ ",city_" + index + "," + (10000.00 * index).toString +
",school_" + index + ":school_" + index + index + "$" + index)
} else {
- socketWriter.println(index.toString + ",name_" + index
+ stringBuilder.append(index.toString + ",name_" + index
+ ",city_" + index + "," + (10000.00 * index).toString +
",school_" + index + ":school_" + index + index + "$" + index)
}
} else {
- socketWriter.println(index.toString + ",name_" + index
+ stringBuilder.append(index.toString + ",name_" + index
+ ",city_" + index + "," + (10000.00 * index).toString +
",school_" + index + ":school_" + index + index + "$" + index)
}
+ stringBuilder.append("\n")
}
+ socketWriter.append(stringBuilder.toString())
socketWriter.flush()
Thread.sleep(1000 * intervalSecond)
}
@@ -1137,8 +818,8 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
thread1.start()
thread2.start()
Thread.sleep(continueSeconds * 1000)
- thread1.interrupt()
thread2.interrupt()
+ thread1.interrupt()
} finally {
if (null != server) {
server.close()
[21/50] [abbrv] carbondata git commit: [CARBONDATA-2081] refresh
cache across different session issue and schema after rename is not being
updated are fixed
Posted by ja...@apache.org.
[CARBONDATA-2081] refresh cache across different session issue and schema after rename is not being updated are fixed
Scenario: 1
open spark-sql and beeline.
create main table in spark-sql
create preaggreagate table in beeline.
drop main table in spark-sql.
perform 'show tables' operation . PreAggregate table is still not deleted.
Scenario: 2
perform following operation in same session:
create table t5 (c1 string, c2 int) stored by 'carbondata'
insert into t5 select 'asd',1
alter table t5 rename to t6
create table t5 (c1 string, c2 int,c3 string) stored by 'carbondata'
insert into t5 select 'asd',1,'sdf' (query is failing)
This closes #1862
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c55240d5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c55240d5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c55240d5
Branch: refs/heads/carbonstore
Commit: c55240d58ad50e718a89d7bd06cfa229ba1ba82b
Parents: 3a6136d
Author: rahulforallp <ra...@knoldus.in>
Authored: Thu Jan 25 19:43:22 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Jan 26 11:55:36 2018 +0530
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/CarbonEnv.scala | 19 ++++++++++++++-
.../schema/CarbonAlterTableRenameCommand.scala | 2 ++
.../spark/sql/hive/CarbonFileMetastore.scala | 2 ++
.../spark/sql/hive/CarbonHiveMetaStore.scala | 4 +++-
.../src/main/spark2.2/CarbonSessionState.scala | 25 +++++++-------------
.../vectorreader/AddColumnTestCases.scala | 12 ++++++++++
6 files changed, 45 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c55240d5/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 585fe67..870b1f3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -23,10 +23,11 @@ import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.execution.command.preaaggregate._
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesFunction
-import org.apache.spark.sql.hive._
+import org.apache.spark.sql.hive.{HiveSessionCatalog, _}
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
@@ -111,6 +112,8 @@ object CarbonEnv {
val carbonEnvMap = new ConcurrentHashMap[SparkSession, CarbonEnv]
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+
def getInstance(sparkSession: SparkSession): CarbonEnv = {
if (sparkSession.isInstanceOf[CarbonSession]) {
sparkSession.sessionState.catalog.asInstanceOf[CarbonSessionCatalog].carbonEnv
@@ -154,6 +157,7 @@ object CarbonEnv {
databaseNameOp: Option[String],
tableName: String)
(sparkSession: SparkSession): CarbonTable = {
+ refreshRelationFromCache(TableIdentifier(tableName, databaseNameOp))(sparkSession)
val databaseName = getDatabaseName(databaseNameOp)(sparkSession)
val catalog = getInstance(sparkSession).carbonMetastore
// refresh cache
@@ -168,6 +172,19 @@ object CarbonEnv {
.carbonTable)
}
+ def refreshRelationFromCache(identifier: TableIdentifier)(sparkSession: SparkSession): Boolean = {
+ var isRefreshed = false
+ val carbonEnv = getInstance(sparkSession)
+ if (carbonEnv.carbonMetastore.checkSchemasModifiedTimeAndReloadTable(identifier)) {
+ sparkSession.sessionState.catalog.refreshTable(identifier)
+ DataMapStoreManager.getInstance().
+ clearDataMaps(AbsoluteTableIdentifier.from(CarbonProperties.getStorePath,
+ identifier.database.getOrElse("default"), identifier.table))
+ isRefreshed = true
+ }
+ isRefreshed
+ }
+
/**
* Return carbon table instance by looking up table in `sparkSession`
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c55240d5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index 64d1d6f..dd34f08 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -122,6 +122,8 @@ private[sql] case class CarbonAlterTableRenameCommand(
metastore.removeTableFromMetadata(oldDatabaseName, oldTableName)
val hiveClient = sparkSession.sessionState.catalog.asInstanceOf[CarbonSessionCatalog]
.getClient()
+ sparkSession.catalog.refreshTable(TableIdentifier(oldTableName,
+ Some(oldDatabaseName)).quotedString)
hiveClient.runSqlHive(
s"ALTER TABLE $oldDatabaseName.$oldTableName RENAME TO $oldDatabaseName.$newTableName")
hiveClient.runSqlHive(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c55240d5/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index 0c52100..b44dc7e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -221,6 +221,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
val schemaConverter = new ThriftWrapperSchemaConverterImpl
val wrapperTableInfo =
schemaConverter.fromExternalToWrapperTableInfo(tableInfo, dbName, tableName, tablePath)
+ CarbonMetadata.getInstance().removeTable(tableUniqueName)
CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
metadata.carbonTables += carbonTable
@@ -258,6 +259,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
oldTableIdentifier.getTableId)
val path = createSchemaThriftFile(newAbsoluteTableIdentifier, thriftTableInfo)
addTableCache(wrapperTableInfo, newAbsoluteTableIdentifier)
+
path
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c55240d5/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
index 54f58fc..759471b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
@@ -51,13 +51,15 @@ class CarbonHiveMetaStore extends CarbonFileMetastore {
absIdentifier: AbsoluteTableIdentifier,
sparkSession: SparkSession): CarbonRelation = {
val info = CarbonUtil.convertGsonToTableInfo(parameters.asJava)
- if (info != null) {
+ val carbonRelation = if (info != null) {
val table = CarbonTable.buildFromTableInfo(info)
CarbonRelation(info.getDatabaseName, info.getFactTable.getTableName,
CarbonSparkUtil.createSparkMeta(table), table)
} else {
super.createCarbonRelation(parameters, absIdentifier, sparkSession)
}
+ carbonRelation.refresh()
+ carbonRelation
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c55240d5/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
index 66a20ea..3c151f0 100644
--- a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
@@ -95,18 +95,6 @@ class CarbonSessionCatalog(
CarbonEnv.initListeners()
- private def refreshRelationFromCache(identifier: TableIdentifier): Boolean = {
- var isRefreshed = false
- if (carbonEnv.carbonMetastore.checkSchemasModifiedTimeAndReloadTable(identifier)) {
- refreshTable(identifier)
- DataMapStoreManager.getInstance().
- clearDataMaps(AbsoluteTableIdentifier.from(CarbonProperties.getStorePath,
- identifier.database.getOrElse("default"), identifier.table))
- logInfo(s"Schema changes have been detected for table: $identifier")
- isRefreshed = true
- }
- isRefreshed
- }
override def lookupRelation(name: TableIdentifier): LogicalPlan = {
@@ -115,17 +103,20 @@ class CarbonSessionCatalog(
rtnRelation match {
case SubqueryAlias(_,
LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
- toRefreshRelation = refreshRelationFromCache(name)
+ toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
case LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _) =>
- toRefreshRelation = refreshRelationFromCache(name)
+ toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
case SubqueryAlias(_, relation) if
relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.HiveTableRelation") ||
relation.getClass.getName.equals(
"org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation") =>
- val catalogTable = CarbonReflectionUtils.getFieldOfCatalogTable("tableMeta",
- relation).asInstanceOf[CatalogTable]
- toRefreshRelation = refreshRelationFromCache(catalogTable.identifier)
+ val catalogTable =
+ CarbonReflectionUtils.getFieldOfCatalogTable(
+ "tableMeta",
+ relation).asInstanceOf[CatalogTable]
+ toRefreshRelation =
+ CarbonEnv.refreshRelationFromCache(catalogTable.identifier)(sparkSession)
case _ =>
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c55240d5/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
index 9a87653..d36dd26 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
@@ -669,6 +669,18 @@ class AddColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll {
sql("drop table if exists renameTextFileTable")
}
+ test("test rename [create table, rename, create same table with different schema]"){
+ sql("drop table if exists t5")
+ sql("drop table if exists t6")
+
+ sql("create table t5 (c1 string, c2 int) stored by 'carbondata'")
+ sql("insert into t5 select 'asd',1")
+ sql("alter table t5 rename to t6")
+ sql("create table t5 (c1 string, c2 int,c3 string) stored by 'carbondata'")
+ sql("insert into t5 select 'asd',1,'sdf'")
+ checkAnswer(sql("select * from t5"),Seq(Row("asd",1,"sdf")))
+ }
+
override def afterAll {
sql("DROP TABLE IF EXISTS addcolumntest")
sql("DROP TABLE IF EXISTS hivetable")
[07/50] [abbrv] carbondata git commit: [CARBONDATA-2057] Support
specifying path when creating pre-aggregate table
Posted by ja...@apache.org.
[CARBONDATA-2057] Support specifying path when creating pre-aggregate table
When creating datamap of pre-aggreagate table, user should be able to specify the persistence location of it.
User can use path property:
This closes #1835
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9882a74c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9882a74c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9882a74c
Branch: refs/heads/carbonstore
Commit: 9882a74c8843085cc13756a03025f163829e194e
Parents: aac7af7
Author: Jacky Li <ja...@qq.com>
Authored: Fri Jan 19 14:48:36 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Jan 19 15:49:01 2018 +0530
----------------------------------------------------------------------
.../testsuite/datamap/TestDataMapCommand.scala | 27 +++++++++++++
.../datamap/CarbonDropDataMapCommand.scala | 40 ++++++++------------
.../CreatePreAggregateTableCommand.scala | 5 ++-
3 files changed, 47 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9882a74c/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index d0a342b..f3458e2 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -17,6 +17,8 @@
package org.apache.carbondata.spark.testsuite.datamap
+import java.io.{File, FilenameFilter}
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
@@ -24,6 +26,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
@@ -207,6 +210,30 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
}
+ test("create pre-agg table with path") {
+ sql("drop table if exists main_preagg")
+ sql("drop table if exists main ")
+ val path = "./_pre-agg_test"
+ sql("create table main(year int,month int,name string,salary int) stored by 'carbondata' tblproperties('sort_columns'='month,year,name')")
+ sql("insert into main select 10,11,'amy',12")
+ sql("insert into main select 10,11,'amy',14")
+ sql("create datamap preagg on table main " +
+ "using 'preaggregate' " +
+ s"dmproperties ('path'='$path') " +
+ "as select name,avg(salary) from main group by name")
+ assertResult(true)(new File(path).exists())
+ assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
+ .list(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.contains(CarbonCommonConstants.FACT_FILE_EXT)
+ }
+ }).length > 0)
+ checkAnswer(sql("select name,avg(salary) from main group by name"), Row("amy", 13.0))
+ checkAnswer(sql("select * from main_preagg"), Row("amy", 26, 2))
+ sql("drop datamap preagg on table main")
+ assertResult(false)(new File(path).exists())
+ sql("drop table main")
+ }
override def afterAll {
sql("DROP TABLE IF EXISTS maintable")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9882a74c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 59aa322..e545b0b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -51,10 +51,11 @@ case class CarbonDropDataMapCommand(
tableName: String)
extends AtomicRunnableCommand {
+ var commandToRun: CarbonDropTableCommand = _
+
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
val dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
- val identifier = TableIdentifier(tableName, Option(dbName))
val locksToBeAcquired = List(LockUsage.METADATA_LOCK)
val carbonEnv = CarbonEnv.getInstance(sparkSession)
val catalog = carbonEnv.carbonMetastore
@@ -68,18 +69,12 @@ case class CarbonDropDataMapCommand(
lock => carbonLocks += CarbonLockUtil.getLockObject(tableIdentifier, lock)
}
LOGGER.audit(s"Deleting datamap [$dataMapName] under table [$tableName]")
- var carbonTable: Option[CarbonTable] =
- catalog.getTableFromMetadataCache(dbName, tableName)
- if (carbonTable.isEmpty) {
- try {
- carbonTable = Some(catalog.lookupRelation(identifier)(sparkSession)
- .asInstanceOf[CarbonRelation].metaData.carbonTable)
- } catch {
- case ex: NoSuchTableException =>
- if (!ifExistsSet) {
- throw ex
- }
- }
+ val carbonTable: Option[CarbonTable] = try {
+ Some(CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession))
+ } catch {
+ case ex: NoSuchTableException =>
+ if (!ifExistsSet) throw ex
+ else None
}
if (carbonTable.isDefined && carbonTable.get.getTableInfo.getDataMapSchemaList.size() > 0) {
val dataMapSchema = carbonTable.get.getTableInfo.getDataMapSchemaList.asScala.zipWithIndex.
@@ -104,12 +99,12 @@ case class CarbonDropDataMapCommand(
tableName))(sparkSession)
if (dataMapSchema.isDefined) {
if (dataMapSchema.get._1.getRelationIdentifier != null) {
- CarbonDropTableCommand(
+ commandToRun = CarbonDropTableCommand(
ifExistsSet = true,
Some(dataMapSchema.get._1.getRelationIdentifier.getDatabaseName),
dataMapSchema.get._1.getRelationIdentifier.getTableName,
- dropChildTable = true
- ).processMetadata(sparkSession)
+ dropChildTable = true)
+ commandToRun.processMetadata(sparkSession)
}
}
// fires the event after dropping datamap from main table schema
@@ -143,14 +138,11 @@ case class CarbonDropDataMapCommand(
override def processData(sparkSession: SparkSession): Seq[Row] = {
// delete the table folder
- val tableIdentifier = CarbonEnv.getIdentifier(databaseNameOp, tableName)(sparkSession)
- DataMapStoreManager.getInstance().clearDataMap(tableIdentifier, dataMapName)
- CarbonDropTableCommand(
- ifExistsSet = true,
- databaseNameOp,
- dataMapName,
- dropChildTable = true
- ).processData(sparkSession)
+ if (commandToRun != null) {
+ DataMapStoreManager.getInstance().clearDataMap(
+ commandToRun.carbonTable.getAbsoluteTableIdentifier, dataMapName)
+ commandToRun.processData(sparkSession)
+ }
Seq.empty
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9882a74c/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index 8b11548..933bf91 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -105,8 +105,11 @@ case class CreatePreAggregateTableCommand(
}
tableModel.parentTable = Some(parentTable)
tableModel.dataMapRelation = Some(fieldRelationMap)
- val tablePath =
+ val tablePath = if (dmProperties.contains("path")) {
+ dmProperties("path")
+ } else {
CarbonEnv.getTablePath(tableModel.databaseNameOp, tableModel.tableName)(sparkSession)
+ }
CarbonCreateTableCommand(TableNewProcessor(tableModel),
tableModel.ifNotExistsSet, Some(tablePath)).run(sparkSession)
[09/50] [abbrv] carbondata git commit: [CARBONDATA-2001] Unable to
Save DataFrame As Carbondata stream table
Posted by ja...@apache.org.
[CARBONDATA-2001] Unable to Save DataFrame As Carbondata stream table
1.added table property for streaming in carbondataframewriter
2.added test case for same
This closes #1774
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bef6af30
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bef6af30
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bef6af30
Branch: refs/heads/carbonstore
Commit: bef6af30e5959a6edbaf95cd004da90ad1e6d646
Parents: bcc9cf0
Author: anubhav100 <an...@knoldus.in>
Authored: Mon Jan 8 13:17:36 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Jan 19 21:27:58 2018 +0800
----------------------------------------------------------------------
.../testsuite/dataload/TestLoadDataFrame.scala | 18 +++++++++++++++++-
.../apache/spark/sql/CarbonDataFrameWriter.scala | 12 +++++++++---
2 files changed, 26 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bef6af30/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index 574eb91..6f03493 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -73,6 +73,7 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS carbon8")
sql("DROP TABLE IF EXISTS carbon9")
sql("DROP TABLE IF EXISTS carbon10")
+ sql("DROP TABLE IF EXISTS carbon11")
sql("DROP TABLE IF EXISTS df_write_sort_column_not_specified")
sql("DROP TABLE IF EXISTS df_write_specify_sort_column")
sql("DROP TABLE IF EXISTS df_write_empty_sort_column")
@@ -244,7 +245,22 @@ test("test the boolean data type"){
.message
.contains("not found"))
}
-
+ test("test streaming Table") {
+ dataFrame.write
+ .format("carbondata")
+ .option("tableName", "carbon11")
+ .option("tempCSV", "true")
+ .option("single_pass", "false")
+ .option("compress", "false")
+ .option("streaming", "true")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("SELECT decimal FROM carbon11"),Seq(Row(BigDecimal.valueOf(10000.00)),Row(BigDecimal.valueOf(1234.44))))
+ val descResult =sql("desc formatted carbon11")
+ val isStreaming: String = descResult.collect().find(row=>row(0).asInstanceOf[String].trim.equalsIgnoreCase("streaming")).get.get(1).asInstanceOf[String]
+ assert(isStreaming.contains("true"))
+ }
private def getSortColumnValue(tableName: String): Array[String] = {
val desc = sql(s"desc formatted $tableName")
val sortColumnRow = desc.collect.find(r =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bef6af30/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index d50f0b8..2b06375 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -167,19 +167,25 @@ class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame) {
val carbonSchema = schema.map { field =>
s"${ field.name } ${ convertToCarbonType(field.dataType) }"
}
+
val property = Map(
"SORT_COLUMNS" -> options.sortColumns,
"DICTIONARY_INCLUDE" -> options.dictionaryInclude,
"DICTIONARY_EXCLUDE" -> options.dictionaryExclude,
- "TABLE_BLOCKSIZE" -> options.tableBlockSize
- ).filter(_._2.isDefined).map(p => s"'${p._1}' = '${p._2.get}'").mkString(",")
+ "TABLE_BLOCKSIZE" -> options.tableBlockSize,
+ "STREAMING" -> Option(options.isStreaming.toString)
+ ).filter(_._2.isDefined)
+ .map(property => s"'${property._1}' = '${property._2.get}'").mkString(",")
+
val dbName = CarbonEnv.getDatabaseName(options.dbName)(sqlContext.sparkSession)
+
s"""
| CREATE TABLE IF NOT EXISTS $dbName.${options.tableName}
| (${ carbonSchema.mkString(", ") })
| STORED BY 'carbondata'
- | ${ if (property.nonEmpty) "TBLPROPERTIES (" + property + ")" else "" }
| ${ if (options.tablePath.nonEmpty) s"LOCATION '${options.tablePath.get}'" else ""}
+ | ${ if (property.nonEmpty) "TBLPROPERTIES (" + property + ")" else "" }
+ |
""".stripMargin
}
[02/50] [abbrv] carbondata git commit: [CARBONDATA-2039] Add relative
blocklet id during initialization in the blocklet data map
Posted by ja...@apache.org.
[CARBONDATA-2039] Add relative blocklet id during initialization in the blocklet data map
Problem
Currently while loading the blocklets in data map all the blocklets are stored in the unsafe manager on array indexes for all the blocklets in one segment. So lets say if 1 segment has 7 task and each task has 10 part files with 3 blocklets each, total number of blocklets in the segment would be 210. Unsafe store will store all these blocklets in one array with start index as 0 and end index as 210.
Due to this while filling the blocklet information the blocklet id taken is the array index which can be any number from 0 to 210. This is leading to loss in actual mapping of blocklet with respect to carbondata file.
Solution
Add the relative blocklet id during loading of blocklets in the unsafe store (relative blocklet id is the id of blocklet in the carbondata file)
This closes #1796
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5ea538fe
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5ea538fe
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5ea538fe
Branch: refs/heads/carbonstore
Commit: 5ea538fe523749aa7f22629cd17a36c7a3e73b90
Parents: d1d726a
Author: manishgupta88 <to...@gmail.com>
Authored: Fri Jan 12 18:28:56 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Jan 18 22:36:00 2018 +0530
----------------------------------------------------------------------
.../blockletindex/BlockletDataMap.java | 39 ++++++++++++++++----
1 file changed, 32 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ea538fe/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index e1fa686..7b2c016 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -101,6 +101,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
private static int LOCATIONS = 10;
+ private static int BLOCKLET_ID_INDEX = 11;
+
private static int TASK_MIN_VALUES_INDEX = 0;
private static int TASK_MAX_VALUES_INDEX = 1;
@@ -130,6 +132,10 @@ public class BlockletDataMap implements DataMap, Cacheable {
isPartitionedSegment = blockletDataMapInfo.isPartitionedSegment();
DataMapRowImpl summaryRow = null;
byte[] schemaBinary = null;
+ // below 2 variables will be used for fetching the relative blocklet id. Relative blocklet ID
+ // is id assigned to a blocklet within a part file
+ String tempFilePath = null;
+ int relativeBlockletId = 0;
for (DataFileFooter fileFooter : indexInfo) {
if (segmentProperties == null) {
List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
@@ -153,9 +159,17 @@ public class BlockletDataMap implements DataMap, Cacheable {
loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
locations);
} else {
+ // blocklet ID will start from 0 again only when part file path is changed
+ if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
+ tempFilePath = blockInfo.getFilePath();
+ relativeBlockletId = 0;
+ }
summaryRow =
loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
- locations);
+ locations, relativeBlockletId);
+ // this is done because relative blocklet id need to be incremented based on the
+ // total number of blocklets
+ relativeBlockletId += fileFooter.getBlockletList().size();
}
}
}
@@ -176,7 +190,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter,
SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
- String[] locations) {
+ String[] locations, int relativeBlockletId) {
int[] minMaxLen = segmentProperties.getColumnsValueSize();
List<BlockletInfo> blockletList = fileFooter.getBlockletList();
CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
@@ -236,7 +250,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
// Add block footer offset, it is used if we need to read footer of block
row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
setLocations(locations, row, ordinal);
-
+ ordinal++;
+ // for relative blockelt id i.e blocklet id that belongs to a particular part file
+ row.setShort((short) relativeBlockletId++, ordinal);
unsafeMemoryDMStore.addIndexRowToUnsafe(row);
} catch (Exception e) {
throw new RuntimeException(e);
@@ -312,6 +328,11 @@ public class BlockletDataMap implements DataMap, Cacheable {
row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
try {
setLocations(locations, row, ordinal);
+ ordinal++;
+ // for relative blocklet id. Value is -1 because in case of old store blocklet info will
+ // not be present in the index file and in that case we will not knwo the total number of
+ // blocklets
+ row.setShort((short) -1, ordinal);
unsafeMemoryDMStore.addIndexRowToUnsafe(row);
} catch (Exception e) {
throw new RuntimeException(e);
@@ -511,6 +532,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
// for locations
indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
+ // for relative blocklet id i.e. blocklet id that belongs to a particular part file.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
+
unsafeMemoryDMStore =
new UnsafeMemoryDMStore(indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]));
}
@@ -628,7 +652,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
int rowCount = unsafeMemoryDMStore.getRowCount();
for (int i = 0; i < rowCount; i++) {
DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(i).convertToSafeRow();
- blocklets.add(createBlocklet(safeRow, i));
+ blocklets.add(createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX)));
}
} else {
int startIndex = findStartIndex(convertToRow(searchStartKey), comparator);
@@ -637,13 +661,14 @@ public class BlockletDataMap implements DataMap, Cacheable {
FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
while (startIndex <= endIndex) {
DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(startIndex).convertToSafeRow();
+ int blockletId = safeRow.getShort(BLOCKLET_ID_INDEX);
String filePath = new String(safeRow.getByteArray(FILE_PATH_INDEX),
CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
boolean isValid =
addBlockBasedOnMinMaxValue(filterExecuter, getMinMaxValue(safeRow, MAX_VALUES_INDEX),
- getMinMaxValue(safeRow, MIN_VALUES_INDEX), filePath, startIndex);
+ getMinMaxValue(safeRow, MIN_VALUES_INDEX), filePath, blockletId);
if (isValid) {
- blocklets.add(createBlocklet(safeRow, startIndex));
+ blocklets.add(createBlocklet(safeRow, blockletId));
}
startIndex++;
}
@@ -712,7 +737,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
public ExtendedBlocklet getDetailedBlocklet(String blockletId) {
int index = Integer.parseInt(blockletId);
DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(index).convertToSafeRow();
- return createBlocklet(safeRow, index);
+ return createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX));
}
private byte[][] getMinMaxValue(DataMapRow row, int index) {
[38/50] [abbrv] carbondata git commit:
[CARBONDATA-2088][CARBONDATA-1516] Optimize syntax for creating timeseries
pre-aggregate table
Posted by ja...@apache.org.
[CARBONDATA-2088][CARBONDATA-1516] Optimize syntax for creating timeseries pre-aggregate table
change using 'timeseries' instead of using preaggregate for creating timeseries pre-aggregate table
change timeseries.eventTime and hour_granularity and so on
granularity only support one
It should throw UnsupportDataMapException if don't use timeseries or preaggregate to create datamap
This closes #1865
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/181c280b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/181c280b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/181c280b
Branch: refs/heads/carbonstore
Commit: 181c280b7d33ac5e4029bd935d6260b0fe79a2bf
Parents: b421c24
Author: xubo245 <60...@qq.com>
Authored: Fri Jan 26 17:47:46 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jan 30 17:25:19 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 -
.../schema/datamap/DataMapProvider.java | 32 +++
.../metadata/schema/datamap/Granularity.java | 46 +++
.../preaggregate/TestPreAggCreateCommand.scala | 60 +++-
.../TestPreAggregateTableSelection.scala | 26 +-
.../timeseries/TestTimeSeriesCreateTable.scala | 280 ++++++++++++++++---
.../timeseries/TestTimeseriesCompaction.scala | 67 ++++-
.../timeseries/TestTimeseriesDataLoad.scala | 142 +++++++++-
.../TestTimeseriesTableSelection.scala | 114 +++++++-
.../testsuite/datamap/TestDataMapCommand.scala | 155 +++++-----
.../MalformedDataMapCommandException.java | 32 +++
.../datamap/CarbonCreateDataMapCommand.scala | 72 ++---
.../CreatePreAggregateTableCommand.scala | 4 +-
.../command/timeseries/TimeSeriesUtil.scala | 83 +++++-
.../carbondata/CarbonDataSourceSuite.scala | 8 +
15 files changed, 943 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index f46feef..cf95dd9 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1543,10 +1543,6 @@ public final class CarbonCommonConstants {
*/
public static final long HANDOFF_SIZE_DEFAULT = 1024L * 1024 * 1024;
- public static final String TIMESERIES_EVENTTIME = "timeseries.eventtime";
-
- public static final String TIMESERIES_HIERARCHY = "timeseries.hierarchy";
-
/**
* It allows queries on hive metastore directly along with filter information, otherwise first
* fetches all partitions from hive and apply filters on it.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
new file mode 100644
index 0000000..65578b1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.datamap;
+
+/**
+ * type for create datamap
+ * The syntax of datamap creation is as follows.
+ * CREATE DATAMAP IF NOT EXISTS dataMapName ON TABLE tableName USING 'DataMapProvider'
+ * DMPROPERTIES('KEY'='VALUE') AS SELECT COUNT(COL1) FROM tableName
+ *
+ * Please refer {{org.apache.spark.sql.parser.CarbonSpark2SqlParser}}
+ */
+
+public enum DataMapProvider {
+ PREAGGREGATE,
+ TIMESERIES;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/Granularity.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/Granularity.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/Granularity.java
new file mode 100644
index 0000000..d6aefb6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/Granularity.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.datamap;
+
+/**
+ * type for create datamap
+ * The syntax of datamap creation is as follows.
+ * CREATE DATAMAP IF NOT EXISTS dataMapName ON TABLE tableName USING 'DataMapProvider'
+ * DMPROPERTIES('KEY'='VALUE') AS SELECT COUNT(COL1) FROM tableName
+ *
+ * Please refer {{org.apache.spark.sql.parser.CarbonSpark2SqlParser}}
+ */
+
+public enum Granularity {
+ YEAR("year_granularity"),
+ MONTH("month_granularity"),
+ DAY("day_granularity"),
+ HOUR("hour_granularity"),
+ MINUTE("minute_granularity"),
+ SECOND("second_granularity");
+ private String name;
+
+ Granularity(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index 755a449..d3f0ff8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -1,15 +1,18 @@
package org.apache.carbondata.integration.spark.testsuite.preaggregate
+import scala.collection.JavaConverters._
+
import org.apache.spark.sql.CarbonDatasourceHadoopRelation
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import scala.collection.JavaConverters._
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
@@ -212,6 +215,60 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
sql("drop datamap agg0 on table maintable")
}
+ val timeSeries = TIMESERIES.toString
+
+ test("test PreAggregate table selection: create with preaggregate and hierarchy") {
+ sql("DROP TABLE IF EXISTS maintabletime")
+ sql(
+ """
+ | CREATE TABLE maintabletime(year INT,month INT,name STRING,salary INT,dob STRING)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES(
+ | 'SORT_SCOPE'='Global_sort',
+ | 'TABLE_BLOCKSIZE'='23',
+ | 'SORT_COLUMNS'='month,year,name')
+ """.stripMargin)
+ sql("INSERT INTO maintabletime SELECT 10,11,'x',12,'2014-01-01 00:00:00'")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0 ON TABLE maintabletime
+ | USING 'preaggregate'
+ | AS SELECT dob,name FROM maintabletime
+ | GROUP BY dob,name
+ """.stripMargin)
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg1 ON TABLE maintabletime
+ | USING 'preaggregate'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dob',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dob,name FROM maintabletime
+ | GROUP BY dob,name
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains(s"$timeSeries keyword missing"))
+ sql("DROP TABLE IF EXISTS maintabletime")
+ }
+
+ test("test pre agg create table 21: using") {
+ sql("DROP DATAMAP agg0 ON TABLE maintable")
+
+ val e: Exception = intercept[Exception] {
+ sql(
+ """
+ | CREATE DATAMAP agg0 ON TABLE mainTable
+ | USING 'abc'
+ | AS SELECT column3, SUM(column3),column5, SUM(column5)
+ | FROM maintable
+ | GROUP BY column3,column5,column2
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains(
+ s"Unknown data map type abc"))
+ sql("DROP DATAMAP agg0 ON TABLE maintable")
+ }
def getCarbontable(plan: LogicalPlan) : CarbonTable ={
var carbonTable : CarbonTable = null
@@ -239,5 +296,6 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists PreAggMain")
sql("drop table if exists PreAggMain1")
sql("drop table if exists PreAggMain2")
+ sql("drop table if exists maintabletime")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 17d95ef..f9ac354 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -23,6 +23,8 @@ import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+
class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
@@ -267,6 +269,8 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
preAggTableValidator(df.queryExecution.analyzed, "maintable")
}
+ val timeSeries = TIMESERIES.toString
+
test("test PreAggregate table selection with timeseries and normal together") {
sql("drop table if exists maintabletime")
sql(
@@ -277,17 +281,26 @@ test("test PreAggregate table selection with timeseries and normal together") {
sql(
"create datamap agg0 on table maintabletime using 'preaggregate' as select dob,name from " +
"maintabletime group by dob,name")
- sql(
- "create datamap agg1 on table maintabletime using 'preaggregate' DMPROPERTIES ('timeseries" +
- ".eventTime'='dob', 'timeseries.hierarchy'='hour=1,day=1,month=1,year=1') as select dob," +
- "name from maintabletime group by dob,name")
- val df = sql("select timeseries(dob,'year') from maintabletime group by timeseries(dob,'year')")
+
+ sql(
+ s"""
+ | CREATE DATAMAP agg1_year ON TABLE maintabletime
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dob',
+ | 'YEAR_GRANULARITY'='1')
+ | AS SELECT dob, name FROM maintabletime
+ | GROUP BY dob,name
+ """.stripMargin)
+
+ val df = sql("SELECT timeseries(dob,'year') FROM maintabletime GROUP BY timeseries(dob,'year')")
preAggTableValidator(df.queryExecution.analyzed, "maintabletime_agg1_year")
+ sql("DROP TABLE IF EXISTS maintabletime")
}
test("test table selection when unsupported aggregate function is present") {
- sql("drop table if exists maintabletime")
+ sql("DROP TABLE IF EXISTS maintabletime")
sql(
"create table maintabletime(year int,month int,name string,salary int,dob string) stored" +
" by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23'," +
@@ -302,6 +315,7 @@ test("test PreAggregate table selection with timeseries and normal together") {
override def afterAll: Unit = {
sql("drop table if exists mainTable")
sql("drop table if exists lineitem")
+ sql("DROP TABLE IF EXISTS maintabletime")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
index c9041fa..0ca7cb9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
@@ -17,14 +17,68 @@
package org.apache.carbondata.integration.spark.testsuite.timeseries
import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.{BeforeAndAfterAll, Ignore}
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.spark.exception.{MalformedDataMapCommandException, MalformedCarbonCommandException}
class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
+ val timeSeries = TIMESERIES.toString
+
override def beforeAll: Unit = {
- sql("drop table if exists mainTable")
+ sql("DROP TABLE IF EXISTS mainTable")
sql("CREATE TABLE mainTable(dataTime timestamp, name string, city string, age int) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='second=1,hour=1,day=1,month=1,year=1') as select dataTime, sum(age) from mainTable group by dataTime")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_hour ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_day ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'day_granularity'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'month_granularity'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_year ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'year_granularity'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
test("test timeseries create table Zero") {
@@ -53,28 +107,29 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
intercept[Exception] {
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
| DMPROPERTIES (
- | 'timeseries.eventTime'='dataTime',
- | 'timeseries.hierarchy'='sec=1,hour=1,day=1,month=1,year=1')
- | as select dataTime, sum(age) from mainTable
- | group by dataTime
- """.stripMargin)
+ | 'EVENT_TIME'='dataTime',
+ | 'SEC_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
}
test("test timeseries create table Six") {
intercept[Exception] {
sql(
- """
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
- | DMPROPERTIES ('timeseries.eventTime'='dataTime', 'timeseries.hierarchy'='hour=2')
- | as select dataTime, sum(age) from mainTable
- | group by dataTime
- """.stripMargin)
-
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='2')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
}
@@ -82,24 +137,24 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
intercept[Exception] {
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
| DMPROPERTIES (
- | 'timeseries.eventTime'='dataTime',
- | 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
- | as select dataTime, sum(age) from mainTable
- | group by dataTime
- """.stripMargin)
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
| DMPROPERTIES (
- | 'timeseries.eventTime'='dataTime',
- | 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
- | as select dataTime, sum(age) from mainTable
- | group by dataTime
- """.stripMargin)
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
}
@@ -107,12 +162,14 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
intercept[Exception] {
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
- | DMPROPERTIES ('timeseries.eventTime'='name', 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
- | as select name, sum(age) from mainTable
- | group by name
- """.stripMargin)
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='name',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
}
@@ -120,18 +177,149 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
intercept[Exception] {
sql(
s"""
- | create datamap agg0 on table mainTable
- | using 'preaggregate'
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='name',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ }
+
+ test("test timeseries create table: USING") {
+ val e: Exception = intercept[MalformedDataMapCommandException] {
+ sql(
+ """CREATE DATAMAP agg1 ON TABLE mainTable
+ | USING 'abc'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Unknown data map type abc"))
+ }
+
+ test("test timeseries create table: USING and catch MalformedCarbonCommandException") {
+ val e: Exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ """CREATE DATAMAP agg1 ON TABLE mainTable
+ | USING 'abc'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Unknown data map type abc"))
+ }
+
+ test("test timeseries create table: Only one granularity level can be defined 1") {
+ val e: Exception = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1',
+ | 'HOUR_GRANULARITY'='1',
+ | 'DAY_GRANULARITY'='1',
+ | 'MONTH_GRANULARITY'='1',
+ | 'YEAR_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Only one granularity level can be defined"))
+ }
+
+ test("test timeseries create table: Only one granularity level can be defined 2") {
+ val e: Exception = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'SECOND_GRANULARITY'='1',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Only one granularity level can be defined"))
+ }
+
+ test("test timeseries create table: Only one granularity level can be defined 3") {
+ val e: Exception = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'DAY_GRANULARITY'='1',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Only one granularity level can be defined"))
+ }
+
+ test("test timeseries create table: Granularity only support 1") {
+ val e = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'DAY_GRANULARITY'='2')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Granularity only support 1"))
+ }
+
+ test("test timeseries create table: Granularity only support 1 and throw Exception") {
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='2')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.equals("Granularity only support 1"))
+ }
+
+ test("test timeseries create table: timeSeries should define time granularity") {
+ val e = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
| DMPROPERTIES (
- | 'timeseries.eventTime'='dataTime',
- | 'timeseries.hierarchy'='hour=1,day=1,year=1,month=1')
- | as select name, sum(age) from mainTable
- | group by name
- """.stripMargin)
+ | 'EVENT_TIME'='dataTime')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
}
+ assert(e.getMessage.equals(s"$timeSeries should define time granularity"))
}
override def afterAll: Unit = {
- sql("drop table if exists mainTable")
+ sql("DROP TABLE IF EXISTS mainTable")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
index a410fe4..d66c402 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesCompaction.scala
@@ -18,15 +18,18 @@ package org.apache.carbondata.integration.spark.testsuite.timeseries
import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.util.SparkUtil4Test
-import org.scalatest.{BeforeAndAfterAll, Ignore}
+import org.scalatest.BeforeAndAfterAll
import org.scalatest.Matchers._
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
import org.apache.carbondata.core.util.CarbonProperties
class TestTimeseriesCompaction extends QueryTest with BeforeAndAfterAll {
var isCompactionEnabled = false
+ val timeSeries = TIMESERIES.toString
+
override def beforeAll: Unit = {
SparkUtil4Test.createTaskMockUp(sqlContext)
CarbonProperties.getInstance()
@@ -38,7 +41,67 @@ class TestTimeseriesCompaction extends QueryTest with BeforeAndAfterAll {
.addProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE, "true")
sql("drop table if exists mainTable")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_minute ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'MINUTE_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_hour ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_day ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'DAY_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_year ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'YEAR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
index d25710c..8bcdfc9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
@@ -24,10 +24,13 @@ import org.apache.spark.util.SparkUtil4Test
import org.scalatest.{BeforeAndAfterAll, Ignore}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
import org.apache.carbondata.core.util.CarbonProperties
class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
+ val timeSeries = TIMESERIES.toString
+
override def beforeAll: Unit = {
SparkUtil4Test.createTaskMockUp(sqlContext)
CarbonProperties.getInstance()
@@ -35,11 +38,131 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists mainTable")
sql("drop table if exists table_03")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_minute ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'minute_granularity'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_hour ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_day ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'DAY_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_year ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'year_granularity'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
sql("CREATE TABLE table_03 (imei string,age int,mac string,productdate timestamp,updatedate timestamp,gamePointId double,contractid double ) STORED BY 'org.apache.carbondata.format'")
sql(s"LOAD DATA inpath '$resourcesPath/data_sort.csv' INTO table table_03 options ('DELIMITER'=',', 'QUOTECHAR'='','FILEHEADER'='imei,age,mac,productdate,updatedate,gamePointId,contractid')")
- sql("create datamap ag1 on table table_03 using 'preaggregate' DMPROPERTIES ( 'timeseries.eventtime'='productdate','timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1')as select productdate,mac,sum(age) from table_03 group by productdate,mac")
+
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_second ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_minute ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'minute_granularity'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_hour ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_day ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'DAY_GRANULARITY'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_month ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'month_granularity'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ag1_year ON TABLE table_03
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='productdate',
+ | 'year_granularity'='1')
+ | AS SELECT productdate,mac,SUM(age) FROM table_03
+ | GROUP BY productdate,mac
+ """.stripMargin)
}
test("test Year level timeseries data validation1 ") {
@@ -93,12 +216,21 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
Row(Timestamp.valueOf("2016-02-23 01:02:50.0"),50)))
}
- test("test if timeseries load is successful on table creation") {
+ test("test if timeseries load is successful ON TABLE creation") {
sql("drop table if exists mainTable")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
- checkAnswer( sql("select * from maintable_agg0_second"),
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ checkAnswer( sql("select * FROM maintable_agg0_second"),
Seq(Row(Timestamp.valueOf("2016-02-23 01:01:30.0"),10),
Row(Timestamp.valueOf("2016-02-23 01:01:40.0"),20),
Row(Timestamp.valueOf("2016-02-23 01:01:50.0"),30),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
index a9d3965..3065952 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
@@ -24,14 +24,78 @@ import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.util.SparkUtil4Test
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
+ val timeSeries = TIMESERIES.toString
+
override def beforeAll: Unit = {
SparkUtil4Test.createTaskMockUp(sqlContext)
sql("drop table if exists mainTable")
sql("CREATE TABLE mainTable(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
- sql("create datamap agg0 on table mainTable using 'preaggregate' DMPROPERTIES ('timeseries.eventTime'='mytime', 'timeseries.hierarchy'='second=1,minute=1,hour=1,day=1,month=1,year=1') as select mytime, sum(age) from mainTable group by mytime")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'SECOND_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_minute ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'minute_granularity'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_hour ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_day ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'DAY_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_month ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'MONTH_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_year ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'YEAR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM mainTable
+ | GROUP BY mytime
+ """.stripMargin)
+
sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
}
@@ -100,6 +164,54 @@ class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
preAggTableValidator(df.queryExecution.analyzed,"maintable")
}
+ test("test timeseries table selection 14: Granularity only support 1 and throw Exception") {
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg3_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='2')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains("Granularity only support 1"))
+ }
+
+ test("test timeseries table selection 15: Granularity only support 1 and throw Exception") {
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg3_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='1.5')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains("Granularity only support 1"))
+ }
+
+ test("test timeseries table selection 16: Granularity only support 1 and throw Exception") {
+ val e = intercept[MalformedCarbonCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP agg3_second ON TABLE mainTable
+ | USING '$timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='dataTime',
+ | 'HOUR_GRANULARITY'='-1')
+ | AS SELECT dataTime, SUM(age) FROM mainTable
+ | GROUP BY dataTime
+ """.stripMargin)
+ }
+ assert(e.getMessage.contains("Granularity only support 1"))
+ }
+
def preAggTableValidator(plan: LogicalPlan, actualTableName: String) : Unit ={
var isValidPlan = false
plan.transform {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index a0ea317..0c38239 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.spark.exception.MalformedDataMapCommandException
class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
@@ -39,37 +40,42 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
sql("create table datamaptest (a string, b string, c string) stored by 'carbondata'")
}
-
- test("test datamap create") {
- sql("create datamap datamap1 on table datamaptest using 'new.class'")
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 1)
- assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap1"))
- assert(dataMapSchemaList.get(0).getClassName.equals("new.class"))
+ val newClass = "org.apache.spark.sql.CarbonSource"
+
+ test("test datamap create: don't support using class, only support short name") {
+ intercept[MalformedDataMapCommandException] {
+ sql(s"CREATE DATAMAP datamap1 ON TABLE datamaptest USING '$newClass'")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 1)
+ assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap1"))
+ assert(dataMapSchemaList.get(0).getClassName.equals(newClass))
+ }
}
- test("test datamap create with dmproperties") {
- sql("create datamap datamap2 on table datamaptest using 'new.class' dmproperties('key'='value')")
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
- assert(dataMapSchemaList.get(1).getDataMapName.equals("datamap2"))
- assert(dataMapSchemaList.get(1).getClassName.equals("new.class"))
- assert(dataMapSchemaList.get(1).getProperties.get("key").equals("value"))
+ test("test datamap create with dmproperties: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 2)
+ assert(dataMapSchemaList.get(1).getDataMapName.equals("datamap2"))
+ assert(dataMapSchemaList.get(1).getClassName.equals(newClass))
+ assert(dataMapSchemaList.get(1).getProperties.get("key").equals("value"))
+ }
}
- test("test datamap create with existing name") {
- intercept[Exception] {
+ test("test datamap create with existing name: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
sql(
- "create datamap datamap2 on table datamaptest using 'new.class' dmproperties('key'='value')")
+ s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 2)
}
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
}
test("test datamap create with preagg") {
@@ -79,10 +85,10 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
assert(table != null)
val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 3)
- assert(dataMapSchemaList.get(2).getDataMapName.equals("datamap3"))
- assert(dataMapSchemaList.get(2).getProperties.get("key").equals("value"))
- assert(dataMapSchemaList.get(2).getChildSchema.getTableName.equals("datamaptest_datamap3"))
+ assert(dataMapSchemaList.size() == 1)
+ assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap3"))
+ assert(dataMapSchemaList.get(0).getProperties.get("key").equals("value"))
+ assert(dataMapSchemaList.get(0).getChildSchema.getTableName.equals("datamaptest_datamap3"))
}
test("check hivemetastore after drop datamap") {
@@ -110,7 +116,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
}
}
- test("drop the table having pre-aggregate"){
+ test("drop the table having pre-aggregate") {
try {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
@@ -140,13 +146,24 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
test("test datamap create with preagg with duplicate name") {
intercept[Exception] {
sql(
- "create datamap datamap2 on table datamaptest using 'preaggregate' dmproperties('key'='value') as select count(a) from datamaptest")
-
+ s"""
+ | CREATE DATAMAP datamap2 ON TABLE datamaptest
+ | USING 'preaggregate'
+ | DMPROPERTIES('key'='value')
+ | AS SELECT COUNT(a) FROM datamaptest
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP datamap2 ON TABLE datamaptest
+ | USING 'preaggregate'
+ | DMPROPERTIES('key'='value')
+ | AS SELECT COUNT(a) FROM datamaptest
+ """.stripMargin)
}
val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
assert(table != null)
val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 3)
+ assert(dataMapSchemaList.size() == 2)
}
test("test datamap drop with preagg") {
@@ -157,25 +174,29 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
assert(table != null)
val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 3)
+ assert(dataMapSchemaList.size() == 2)
}
- test("test show datamap without preaggregate") {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'new.class' dmproperties('key'='value')")
- sql("create datamap datamap2 on table datamapshowtest using 'new.class' dmproperties('key'='value')")
- checkExistence(sql("show datamap on table datamapshowtest"), true, "datamap1", "datamap2", "(NA)", "new.class")
+ test("test show datamap without preaggregate: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql(s"CREATE DATAMAP datamap1 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
+ checkExistence(sql("SHOW DATAMAP ON TABLE datamapshowtest"), true, "datamap1", "datamap2", "(NA)", newClass)
+ }
}
- test("test show datamap with preaggregate") {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
- sql("create datamap datamap2 on table datamapshowtest using 'new.class' dmproperties('key'='value')")
- val frame = sql("show datamap on table datamapshowtest")
- assert(frame.collect().length == 2)
- checkExistence(frame, true, "datamap1", "datamap2", "(NA)", "new.class", "default.datamapshowtest_datamap1")
+ test("test show datamap with preaggregate: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
+ val frame = sql("show datamap on table datamapshowtest")
+ assert(frame.collect().length == 2)
+ checkExistence(frame, true, "datamap1", "datamap2", "(NA)", newClass, "default.datamapshowtest_datamap1")
+ }
}
test("test show datamap with no datamap") {
@@ -184,15 +205,17 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
assert(sql("show datamap on table datamapshowtest").collect().length == 0)
}
- test("test show datamap after dropping datamap") {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
- sql("create datamap datamap2 on table datamapshowtest using 'new.class' dmproperties('key'='value')")
- sql("drop datamap datamap1 on table datamapshowtest")
- val frame = sql("show datamap on table datamapshowtest")
- assert(frame.collect().length == 1)
- checkExistence(frame, true, "datamap2", "(NA)", "new.class")
+ test("test show datamap after dropping datamap: don't support using class") {
+ intercept[MalformedDataMapCommandException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
+ sql("drop datamap datamap1 on table datamapshowtest")
+ val frame = sql("show datamap on table datamapshowtest")
+ assert(frame.collect().length == 1)
+ checkExistence(frame, true, "datamap2", "(NA)", newClass)
+ }
}
test("test if preaggregate load is successfull for hivemetastore") {
@@ -217,7 +240,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
- checkAnswer(sql("select * from uniqdata_uniqdata_agg"),Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
+ checkAnswer(sql("select * from uniqdata_uniqdata_agg"), Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
sql("drop datamap if exists uniqdata_agg on table uniqdata")
}
@@ -229,16 +252,16 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
sql("insert into main select 10,11,'amy',12")
sql("insert into main select 10,11,'amy',14")
sql("create datamap preagg on table main " +
- "using 'preaggregate' " +
- s"dmproperties ('path'='$path') " +
- "as select name,avg(salary) from main group by name")
+ "using 'preaggregate' " +
+ s"dmproperties ('path'='$path') " +
+ "as select name,avg(salary) from main group by name")
assertResult(true)(new File(path).exists())
assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
- .list(new FilenameFilter {
- override def accept(dir: File, name: String): Boolean = {
- name.contains(CarbonCommonConstants.FACT_FILE_EXT)
- }
- }).length > 0)
+ .list(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.contains(CarbonCommonConstants.FACT_FILE_EXT)
+ }
+ }).length > 0)
checkAnswer(sql("select name,avg(salary) from main group by name"), Row("amy", 13.0))
checkAnswer(sql("select * from main_preagg"), Row("amy", 26, 2))
sql("drop datamap preagg on table main")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java
new file mode 100644
index 0000000..a05d8e6
--- /dev/null
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.exception;
+
+/**
+ * Throw exception when using illegal argument
+ */
+public class MalformedDataMapCommandException extends MalformedCarbonCommandException {
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ public MalformedDataMapCommandException(String msg) {
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 8e00635..c4d32b4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -16,18 +16,15 @@
*/
package org.apache.spark.sql.execution.command.datamap
-import scala.collection.JavaConverters._
-
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.command.preaaggregate.{CreatePreAggregateTableCommand, PreAggregateUtil}
+import org.apache.spark.sql.execution.command.preaaggregate.CreatePreAggregateTableCommand
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider._
+import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, MalformedDataMapCommandException}
/**
* Below command class will be used to create datamap on table
@@ -41,70 +38,63 @@ case class CarbonCreateDataMapCommand(
queryString: Option[String])
extends AtomicRunnableCommand {
- var createPreAggregateTableCommands: Seq[CreatePreAggregateTableCommand] = _
+ var createPreAggregateTableCommands: CreatePreAggregateTableCommand = _
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
// since streaming segment does not support building index and pre-aggregate yet,
// so streaming table does not support create datamap
val carbonTable =
- CarbonEnv.getCarbonTable(tableIdentifier.database, tableIdentifier.table)(sparkSession)
+ CarbonEnv.getCarbonTable(tableIdentifier.database, tableIdentifier.table)(sparkSession)
if (carbonTable.isStreamingTable) {
throw new MalformedCarbonCommandException("Streaming table does not support creating datamap")
}
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- if (dmClassName.equals("org.apache.carbondata.datamap.AggregateDataMapHandler") ||
- dmClassName.equalsIgnoreCase("preaggregate")) {
- val timeHierarchyString = dmproperties.get(CarbonCommonConstants.TIMESERIES_HIERARCHY)
- createPreAggregateTableCommands = if (timeHierarchyString.isDefined) {
+
+ if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
+ dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ TimeSeriesUtil.validateTimeSeriesGranularity(dmproperties, dmClassName)
+ createPreAggregateTableCommands = if (dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
val details = TimeSeriesUtil
- .validateAndGetTimeSeriesHierarchyDetails(
- timeHierarchyString.get)
- val updatedDmProperties = dmproperties - CarbonCommonConstants.TIMESERIES_HIERARCHY
- details.map { f =>
- CreatePreAggregateTableCommand(dataMapName + '_' + f._1,
- tableIdentifier,
- dmClassName,
- updatedDmProperties,
- queryString.get,
- Some(f._1))
- }.toSeq
+ .getTimeSeriesGranularityDetails(dmproperties, dmClassName)
+ val updatedDmProperties = dmproperties - details._1
+ CreatePreAggregateTableCommand(dataMapName,
+ tableIdentifier,
+ dmClassName,
+ updatedDmProperties,
+ queryString.get,
+ Some(details._1))
} else {
- Seq(CreatePreAggregateTableCommand(
+ CreatePreAggregateTableCommand(
dataMapName,
tableIdentifier,
dmClassName,
dmproperties,
queryString.get
- ))
+ )
}
- createPreAggregateTableCommands.flatMap(_.processMetadata(sparkSession))
+ createPreAggregateTableCommands.processMetadata(sparkSession)
} else {
- val dataMapSchema = new DataMapSchema(dataMapName, dmClassName)
- dataMapSchema.setProperties(new java.util.HashMap[String, String](dmproperties.asJava))
- val dbName = CarbonEnv.getDatabaseName(tableIdentifier.database)(sparkSession)
- // upadting the parent table about dataschema
- PreAggregateUtil.updateMainTable(dbName, tableIdentifier.table, dataMapSchema, sparkSession)
+ throw new MalformedDataMapCommandException("Unknown data map type " + dmClassName)
}
- LOGGER.audit(s"DataMap $dataMapName successfully added to Table ${ tableIdentifier.table }")
+ LOGGER.audit(s"DataMap $dataMapName successfully added to Table ${tableIdentifier.table}")
Seq.empty
}
override def processData(sparkSession: SparkSession): Seq[Row] = {
- if (dmClassName.equals("org.apache.carbondata.datamap.AggregateDataMapHandler") ||
- dmClassName.equalsIgnoreCase("preaggregate")) {
- createPreAggregateTableCommands.flatMap(_.processData(sparkSession))
+ if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
+ dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ createPreAggregateTableCommands.processData(sparkSession)
} else {
- Seq.empty
+ throw new MalformedDataMapCommandException("Unknown data map type " + dmClassName)
}
}
override def undoMetadata(sparkSession: SparkSession, exception: Exception): Seq[Row] = {
- if (dmClassName.equals("org.apache.carbondata.datamap.AggregateDataMapHandler") ||
- dmClassName.equalsIgnoreCase("preaggregate")) {
- val timeHierarchyString = dmproperties.get(CarbonCommonConstants.TIMESERIES_HIERARCHY)
- createPreAggregateTableCommands.flatMap(_.undoMetadata(sparkSession, exception))
+ if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
+ dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ createPreAggregateTableCommands.undoMetadata(sparkSession, exception)
} else {
- Seq.empty
+ throw new MalformedDataMapCommandException("Unknown data map type " + dmClassName)
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index a75a06f..dbbf90c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -104,9 +104,9 @@ case class CreatePreAggregateTableCommand(
TimeSeriesUtil.validateTimeSeriesEventTime(dmProperties, parentTable)
TimeSeriesUtil.validateEventTimeColumnExitsInSelect(
fieldRelationMap,
- dmProperties.get(CarbonCommonConstants.TIMESERIES_EVENTTIME).get)
+ dmProperties.get(TimeSeriesUtil.TIMESERIES_EVENTTIME).get)
TimeSeriesUtil.updateTimeColumnSelect(fieldRelationMap,
- dmProperties.get(CarbonCommonConstants.TIMESERIES_EVENTTIME).get,
+ dmProperties.get(TimeSeriesUtil.TIMESERIES_EVENTTIME).get,
timeSeriesFunction.get)
}
tableModel.parentTable = Some(parentTable)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
index 4fe9df0..987d4fe 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
@@ -18,32 +18,33 @@ package org.apache.spark.sql.execution.command.timeseries
import org.apache.spark.sql.execution.command.{DataMapField, Field}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
+import org.apache.carbondata.core.metadata.schema.datamap.Granularity
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.preagg.TimeSeriesUDF
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, MalformedDataMapCommandException}
/**
* Utility class for time series to keep
*/
object TimeSeriesUtil {
+ val TIMESERIES_EVENTTIME = "event_time"
+
/**
* Below method will be used to validate whether column mentioned in time series
* is timestamp column or not
*
- * @param dmproperties
- * data map properties
- * @param parentTable
- * parent table
+ * @param dmproperties data map properties
+ * @param parentTable parent table
* @return whether time stamp column
*/
def validateTimeSeriesEventTime(dmproperties: Map[String, String],
parentTable: CarbonTable) {
- val eventTime = dmproperties.get(CarbonCommonConstants.TIMESERIES_EVENTTIME)
+ val eventTime = dmproperties.get(TIMESERIES_EVENTTIME)
if (!eventTime.isDefined) {
- throw new MalformedCarbonCommandException("Eventtime not defined in time series")
+ throw new MalformedCarbonCommandException("event_time not defined in time series")
} else {
val carbonColumn = parentTable.getColumnByName(parentTable.getTableName, eventTime.get)
if (carbonColumn.getDataType != DataTypes.TIMESTAMP) {
@@ -55,13 +56,79 @@ object TimeSeriesUtil {
}
/**
+ * validate TimeSeries Granularity
+ *
+ * @param dmProperties datamap properties
+ * @param dmClassName datamap class name
+ * @return whether find only one granularity
+ */
+ def validateTimeSeriesGranularity(
+ dmProperties: Map[String, String],
+ dmClassName: String): Boolean = {
+ var isFound = false
+
+ // 1. granularity only support one
+ for (granularity <- Granularity.values()) {
+ if (dmProperties.get(granularity.getName).isDefined) {
+ if (isFound) {
+ throw new MalformedDataMapCommandException(
+ s"Only one granularity level can be defined")
+ } else {
+ isFound = true
+ }
+ }
+ }
+
+ // 2. check whether timeseries and granularity match
+ if (isFound && !dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ throw new MalformedDataMapCommandException(
+ s"${TIMESERIES.toString} keyword missing")
+ } else if (!isFound && dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ throw new MalformedDataMapCommandException(
+ s"${TIMESERIES.toString} should define time granularity")
+ } else if (isFound) {
+ true
+ } else {
+ false
+ }
+ }
+
+ /**
+ * get TimeSeries Granularity key and value
+ * check the value
+ *
+ * TODO:we will support value not only equal to 1 in the future
+ *
+ * @param dmProperties datamap properties
+ * @param dmClassName datamap class name
+ * @return key and value tuple
+ */
+ def getTimeSeriesGranularityDetails(
+ dmProperties: Map[String, String],
+ dmClassName: String): (String, String) = {
+
+ val defaultValue = "1"
+ for (granularity <- Granularity.values()) {
+ if (dmProperties.get(granularity.getName).isDefined &&
+ dmProperties.get(granularity.getName).get.equalsIgnoreCase(defaultValue)) {
+ return (granularity.toString.toLowerCase, dmProperties.get(granularity.getName).get)
+ }
+ }
+
+ throw new MalformedDataMapCommandException(
+ s"Granularity only support $defaultValue")
+ }
+
+ /**
* Below method will be used to validate the hierarchy of time series and its value
* validation will be done whether hierarchy order is proper or not and hierarchy level
* value
+ * TODO: we should remove this method
*
* @param timeSeriesHierarchyDetails
* time series hierarchy string
*/
+ @deprecated
def validateAndGetTimeSeriesHierarchyDetails(timeSeriesHierarchyDetails: String): Array[
(String, String)] = {
val updatedtimeSeriesHierarchyDetails = timeSeriesHierarchyDetails.toLowerCase
http://git-wip-us.apache.org/repos/asf/carbondata/blob/181c280b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
index 0f934cb..eb52910 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
@@ -260,4 +260,12 @@ class CarbonDataSourceSuite extends Spark2QueryTest with BeforeAndAfterAll {
sql("drop table if exists carbon_test")
assert(exception.contains("Table creation failed. Table name cannot contain blank space"))
}
+
+ test("test create table: using") {
+ sql("DROP TABLE IF EXISTS usingTable")
+ val e: Exception = intercept[ClassNotFoundException] {
+ sql("CREATE TABLE usingTable(name STRING) USING abc")
+ }
+ assert(e.getMessage.contains("Failed to find data source: abc"))
+ }
}
[16/50] [abbrv] carbondata git commit: [CARBONDATA-2070]fix create
preaggregate on decimal column in hive metastore
Posted by ja...@apache.org.
[CARBONDATA-2070]fix create preaggregate on decimal column in hive metastore
Problem: when hive metastore is enabled and aggregate table is tried to create on the decimal column of main table, cast exception is thrown for Decimal datatype
solution:During creation of TableInfo from hivemetastore the DataMapSchemas and the columns
dataTypes are not converted to the appropriate child classes.
convert to actual datatype
This closes #1852
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ee11bb1e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ee11bb1e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ee11bb1e
Branch: refs/heads/carbonstore
Commit: ee11bb1eb7790687b685b4af35ec9f1ab7fa6eff
Parents: 44ffaf5
Author: akashrn5 <ak...@gmail.com>
Authored: Tue Jan 23 18:59:51 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Jan 24 23:51:30 2018 +0530
----------------------------------------------------------------------
.../core/metadata/schema/table/CarbonTable.java | 11 +++++++++++
.../spark/testsuite/datamap/TestDataMapCommand.scala | 12 ++++++++++++
2 files changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee11bb1e/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index dd65b84..4bb0d20 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -156,6 +156,17 @@ public class CarbonTable implements Serializable {
columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
columnSchema.getPrecision(), columnSchema.getScale()));
}
+ List<DataMapSchema> childSchema = tableInfo.getDataMapSchemaList();
+ for (DataMapSchema dataMapSchema : childSchema) {
+ if (dataMapSchema.childSchema != null
+ && dataMapSchema.childSchema.getListOfColumns().size() > 0) {
+ for (ColumnSchema columnSchema : dataMapSchema.childSchema.getListOfColumns()) {
+ columnSchema.setDataType(DataTypeUtil
+ .valueOf(columnSchema.getDataType(), columnSchema.getPrecision(),
+ columnSchema.getScale()));
+ }
+ }
+ }
if (tableInfo.getFactTable().getBucketingInfo() != null) {
for (ColumnSchema columnSchema : tableInfo.getFactTable()
.getBucketingInfo().getListOfColumns()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee11bb1e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index f3458e2..a0ea317 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -35,6 +35,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
sql("drop table if exists datamaptest")
sql("drop table if exists datamapshowtest")
+ sql("drop table if exists uniqdata")
sql("create table datamaptest (a string, b string, c string) stored by 'carbondata'")
}
@@ -210,6 +211,16 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
}
+ test("test preaggregate load for decimal column for hivemetastore") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
+ sql("drop datamap if exists uniqdata_agg on table uniqdata")
+ sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
+ sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
+ sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
+ checkAnswer(sql("select * from uniqdata_uniqdata_agg"),Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
+ sql("drop datamap if exists uniqdata_agg on table uniqdata")
+ }
+
test("create pre-agg table with path") {
sql("drop table if exists main_preagg")
sql("drop table if exists main ")
@@ -237,6 +248,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
override def afterAll {
sql("DROP TABLE IF EXISTS maintable")
+ sql("drop table if exists uniqdata")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
sql("drop table if exists datamaptest")
[22/50] [abbrv] carbondata git commit: [CARBONDATA-2054]Add an
example: how to use CarbonData batch load to integrate with Spark Streaming
Posted by ja...@apache.org.
[CARBONDATA-2054]Add an example: how to use CarbonData batch load to integrate with Spark Streaming
Use CarbonSession.createDataFrame to convert rdd to DataFrame in DStream.foreachRDD, and then write batch data into CarbonData table which support auto compaction.
This closes #1840
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/09fb83ef
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/09fb83ef
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/09fb83ef
Branch: refs/heads/carbonstore
Commit: 09fb83ef0f9e57db9dcfcc68bec78849a91d9a7f
Parents: c55240d
Author: Zhang Zhichao <44...@qq.com>
Authored: Sun Jan 21 16:36:50 2018 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Jan 26 21:29:33 2018 +0800
----------------------------------------------------------------------
examples/spark2/pom.xml | 6 +
.../CarbonBatchSparkStreamingExample.scala | 216 +++++++++++++++++++
.../CarbonStructuredStreamingExample.scala | 215 ++++++++++++++++++
.../carbondata/examples/ExampleUtils.scala | 15 +-
.../carbondata/examples/StreamExample.scala | 215 ------------------
.../streaming/segment/StreamSegment.java | 2 +-
6 files changed, 448 insertions(+), 221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/09fb83ef/examples/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index 227da7d..da39f1d 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -56,6 +56,12 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-repl_${scala.binary.version}</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark.deps.scope}</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/09fb83ef/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonBatchSparkStreamingExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonBatchSparkStreamingExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonBatchSparkStreamingExample.scala
new file mode 100644
index 0000000..6ae87b9
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonBatchSparkStreamingExample.scala
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.{File, PrintWriter}
+import java.net.ServerSocket
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{CarbonEnv, SaveMode, SparkSession}
+import org.apache.spark.streaming.{Seconds, StreamingContext, Time}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+
+/**
+ * This example introduces how to use CarbonData batch load to integrate
+ * with Spark Streaming(it's DStream, not Spark Structured Streaming)
+ */
+// scalastyle:off println
+
+case class DStreamData(id: Int, name: String, city: String, salary: Float)
+
+object CarbonBatchSparkStreamingExample {
+
+ def main(args: Array[String]): Unit = {
+
+ // setup paths
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val checkpointPath =
+ s"$rootPath/examples/spark2/target/spark_streaming_cp_" +
+ System.currentTimeMillis().toString()
+ val streamTableName = s"dstream_batch_table"
+
+ val spark = ExampleUtils.createCarbonSession("CarbonBatchSparkStreamingExample", 4)
+
+ val requireCreateTable = true
+
+ if (requireCreateTable) {
+ // drop table if exists previously
+ spark.sql(s"DROP TABLE IF EXISTS ${ streamTableName }")
+ // Create target carbon table and populate with initial data
+ // set AUTO_LOAD_MERGE to true to compact segment automatically
+ spark.sql(
+ s"""
+ | CREATE TABLE ${ streamTableName }(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES(
+ | 'sort_columns'='name',
+ | 'dictionary_include'='city',
+ | 'AUTO_LOAD_MERGE'='true',
+ | 'COMPACTION_LEVEL_THRESHOLD'='4,10')
+ | """.stripMargin)
+
+ val carbonTable = CarbonEnv.getCarbonTable(Some("default"), streamTableName)(spark)
+ val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
+ // batch load
+ val path = s"$rootPath/examples/spark2/src/main/resources/streamSample.csv"
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE $streamTableName
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ // streaming ingest
+ val serverSocket = new ServerSocket(7071)
+ val thread1 = writeSocket(serverSocket)
+ val thread2 = showTableCount(spark, streamTableName)
+ val ssc = startStreaming(spark, streamTableName, tablePath, checkpointPath)
+ // wait for stop signal to stop Spark Streaming App
+ waitForStopSignal(ssc)
+ // it need to start Spark Streaming App in main thread
+ // otherwise it will encounter an not-serializable exception.
+ ssc.start()
+ ssc.awaitTermination()
+ thread1.interrupt()
+ thread2.interrupt()
+ serverSocket.close()
+ }
+
+ spark.sql(s"select count(*) from ${ streamTableName }").show(100, truncate = false)
+
+ spark.sql(s"select * from ${ streamTableName }").show(100, truncate = false)
+
+ // record(id = 100000001) comes from batch segment_0
+ // record(id = 1) comes from stream segment_1
+ spark.sql(s"select * " +
+ s"from ${ streamTableName } " +
+ s"where id = 100000001 or id = 1 limit 100").show(100, truncate = false)
+
+ // not filter
+ spark.sql(s"select * " +
+ s"from ${ streamTableName } " +
+ s"where id < 10 limit 100").show(100, truncate = false)
+
+ // show segments
+ spark.sql(s"SHOW SEGMENTS FOR TABLE ${streamTableName}").show(false)
+
+ // drop table
+ spark.sql(s"DROP TABLE IF EXISTS ${ streamTableName }")
+
+ spark.stop()
+ System.out.println("streaming finished")
+ }
+
+ def showTableCount(spark: SparkSession, tableName: String): Thread = {
+ val thread = new Thread() {
+ override def run(): Unit = {
+ for (_ <- 0 to 1000) {
+ spark.sql(s"select count(*) from $tableName").show(truncate = false)
+ spark.sql(s"SHOW SEGMENTS FOR TABLE ${tableName}").show(false)
+ Thread.sleep(1000 * 5)
+ }
+ }
+ }
+ thread.start()
+ thread
+ }
+
+ def waitForStopSignal(ssc: StreamingContext): Thread = {
+ val thread = new Thread() {
+ override def run(): Unit = {
+ // use command 'nc 127.0.0.1 7072' to stop Spark Streaming App
+ new ServerSocket(7072).accept()
+ // don't stop SparkContext here
+ ssc.stop(false, true)
+ }
+ }
+ thread.start()
+ thread
+ }
+
+ def startStreaming(spark: SparkSession, tableName: String,
+ tablePath: CarbonTablePath, checkpointPath: String): StreamingContext = {
+ var ssc: StreamingContext = null
+ try {
+ // recommend: the batch interval must set larger, such as 30s, 1min.
+ ssc = new StreamingContext(spark.sparkContext, Seconds(15))
+ ssc.checkpoint(checkpointPath)
+
+ val readSocketDF = ssc.socketTextStream("localhost", 7071)
+
+ val batchData = readSocketDF
+ .map(_.split(","))
+ .map(fields => DStreamData(fields(0).toInt, fields(1), fields(2), fields(3).toFloat))
+
+ batchData.foreachRDD { (rdd: RDD[DStreamData], time: Time) => {
+ val df = SparkSession.builder().getOrCreate()
+ .createDataFrame(rdd).toDF("id", "name", "city", "salary")
+ println("at time: " + time.toString() + " the count of received data: " + df.count())
+ df.write
+ .format("carbondata")
+ .option("tableName", tableName)
+ .option("tempCSV", "false")
+ .option("compress", "true")
+ .option("single_pass", "true")
+ .mode(SaveMode.Append)
+ .save()
+ }}
+ } catch {
+ case ex: Exception =>
+ ex.printStackTrace()
+ println("Done reading and writing streaming data")
+ }
+ ssc
+ }
+
+ def writeSocket(serverSocket: ServerSocket): Thread = {
+ val thread = new Thread() {
+ override def run(): Unit = {
+ // wait for client to connection request and accept
+ val clientSocket = serverSocket.accept()
+ val socketWriter = new PrintWriter(clientSocket.getOutputStream())
+ var index = 0
+ for (_ <- 1 to 1000) {
+ // write 5 records per iteration
+ for (_ <- 0 to 1000) {
+ index = index + 1
+ socketWriter.println(index.toString + ",name_" + index
+ + ",city_" + index + "," + (index * 10000.00).toString +
+ ",school_" + index + ":school_" + index + index + "$" + index)
+ }
+ socketWriter.flush()
+ Thread.sleep(1000)
+ }
+ socketWriter.close()
+ System.out.println("Socket closed")
+ }
+ }
+ thread.start()
+ thread
+ }
+}
+// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/carbondata/blob/09fb83ef/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStructuredStreamingExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStructuredStreamingExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStructuredStreamingExample.scala
new file mode 100644
index 0000000..247a59b
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStructuredStreamingExample.scala
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.{File, PrintWriter}
+import java.net.ServerSocket
+
+import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+
+// scalastyle:off println
+object CarbonStructuredStreamingExample {
+ def main(args: Array[String]) {
+
+ // setup paths
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+ val storeLocation = s"$rootPath/examples/spark2/target/store"
+ val warehouse = s"$rootPath/examples/spark2/target/warehouse"
+ val metastoredb = s"$rootPath/examples/spark2/target"
+ val streamTableName = s"stream_table"
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ import org.apache.spark.sql.CarbonSession._
+ val spark = SparkSession
+ .builder()
+ .master("local")
+ .appName("CarbonStructuredStreamingExample")
+ .config("spark.sql.warehouse.dir", warehouse)
+ .getOrCreateCarbonSession(storeLocation, metastoredb)
+
+ spark.sparkContext.setLogLevel("ERROR")
+
+ val requireCreateTable = true
+ val useComplexDataType = false
+
+ if (requireCreateTable) {
+ // drop table if exists previously
+ spark.sql(s"DROP TABLE IF EXISTS ${ streamTableName }")
+ // Create target carbon table and populate with initial data
+ if (useComplexDataType) {
+ spark.sql(
+ s"""
+ | CREATE TABLE ${ streamTableName }(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT,
+ | file struct<school:array<string>, age:int>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES(
+ | 'streaming'='true', 'sort_columns'='name', 'dictionary_include'='city')
+ | """.stripMargin)
+ } else {
+ spark.sql(
+ s"""
+ | CREATE TABLE ${ streamTableName }(
+ | id INT,
+ | name STRING,
+ | city STRING,
+ | salary FLOAT
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES(
+ | 'streaming'='true', 'sort_columns'='name')
+ | """.stripMargin)
+ }
+
+ val carbonTable = CarbonEnv.getCarbonTable(Some("default"), streamTableName)(spark)
+ val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
+ // batch load
+ val path = s"$rootPath/examples/spark2/src/main/resources/streamSample.csv"
+ spark.sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path'
+ | INTO TABLE $streamTableName
+ | OPTIONS('HEADER'='true')
+ """.stripMargin)
+
+ // streaming ingest
+ val serverSocket = new ServerSocket(7071)
+ val thread1 = startStreaming(spark, tablePath)
+ val thread2 = writeSocket(serverSocket)
+ val thread3 = showTableCount(spark, streamTableName)
+
+ System.out.println("type enter to interrupt streaming")
+ System.in.read()
+ thread1.interrupt()
+ thread2.interrupt()
+ thread3.interrupt()
+ serverSocket.close()
+ }
+
+ spark.sql(s"select count(*) from ${ streamTableName }").show(100, truncate = false)
+
+ spark.sql(s"select * from ${ streamTableName }").show(100, truncate = false)
+
+ // record(id = 100000001) comes from batch segment_0
+ // record(id = 1) comes from stream segment_1
+ spark.sql(s"select * " +
+ s"from ${ streamTableName } " +
+ s"where id = 100000001 or id = 1 limit 100").show(100, truncate = false)
+
+ // not filter
+ spark.sql(s"select * " +
+ s"from ${ streamTableName } " +
+ s"where id < 10 limit 100").show(100, truncate = false)
+
+ if (useComplexDataType) {
+ // complex
+ spark.sql(s"select file.age, file.school " +
+ s"from ${ streamTableName } " +
+ s"where where file.age = 30 ").show(100, truncate = false)
+ }
+
+ spark.stop()
+ System.out.println("streaming finished")
+ }
+
+ def showTableCount(spark: SparkSession, tableName: String): Thread = {
+ val thread = new Thread() {
+ override def run(): Unit = {
+ for (_ <- 0 to 1000) {
+ spark.sql(s"select count(*) from $tableName").show(truncate = false)
+ Thread.sleep(1000 * 3)
+ }
+ }
+ }
+ thread.start()
+ thread
+ }
+
+ def startStreaming(spark: SparkSession, tablePath: CarbonTablePath): Thread = {
+ val thread = new Thread() {
+ override def run(): Unit = {
+ var qry: StreamingQuery = null
+ try {
+ val readSocketDF = spark.readStream
+ .format("socket")
+ .option("host", "localhost")
+ .option("port", 7071)
+ .load()
+
+ // Write data from socket stream to carbondata file
+ qry = readSocketDF.writeStream
+ .format("carbondata")
+ .trigger(ProcessingTime("5 seconds"))
+ .option("checkpointLocation", tablePath.getStreamingCheckpointDir)
+ .option("dbName", "default")
+ .option("tableName", "stream_table")
+ .start()
+
+ qry.awaitTermination()
+ } catch {
+ case ex: Exception =>
+ ex.printStackTrace()
+ println("Done reading and writing streaming data")
+ } finally {
+ qry.stop()
+ }
+ }
+ }
+ thread.start()
+ thread
+ }
+
+ def writeSocket(serverSocket: ServerSocket): Thread = {
+ val thread = new Thread() {
+ override def run(): Unit = {
+ // wait for client to connection request and accept
+ val clientSocket = serverSocket.accept()
+ val socketWriter = new PrintWriter(clientSocket.getOutputStream())
+ var index = 0
+ for (_ <- 1 to 1000) {
+ // write 5 records per iteration
+ for (_ <- 0 to 1000) {
+ index = index + 1
+ socketWriter.println(index.toString + ",name_" + index
+ + ",city_" + index + "," + (index * 10000.00).toString +
+ ",school_" + index + ":school_" + index + index + "$" + index)
+ }
+ socketWriter.flush()
+ Thread.sleep(1000)
+ }
+ socketWriter.close()
+ System.out.println("Socket closed")
+ }
+ }
+ thread.start()
+ thread
+ }
+}
+// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/carbondata/blob/09fb83ef/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala
index c62a240..fee5e7c 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/ExampleUtils.scala
@@ -32,11 +32,12 @@ object ExampleUtils {
.getCanonicalPath
val storeLocation: String = currentPath + "/target/store"
- def createCarbonSession(appName: String): SparkSession = {
+ def createCarbonSession(appName: String, workThreadNum: Int = 1): SparkSession = {
val rootPath = new File(this.getClass.getResource("/").getPath
+ "../../../..").getCanonicalPath
val storeLocation = s"$rootPath/examples/spark2/target/store"
val warehouse = s"$rootPath/examples/spark2/target/warehouse"
+ val metastoredb = s"$rootPath/examples/spark2/target"
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
@@ -44,14 +45,18 @@ object ExampleUtils {
.addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING, "true")
.addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "")
+ val masterUrl = if (workThreadNum <= 1) {
+ "local"
+ } else {
+ "local[" + workThreadNum.toString() + "]"
+ }
import org.apache.spark.sql.CarbonSession._
val spark = SparkSession
.builder()
- .master("local")
- .appName("CarbonSessionExample")
+ .master(masterUrl)
+ .appName(appName)
.config("spark.sql.warehouse.dir", warehouse)
- .config("spark.driver.host", "localhost")
- .getOrCreateCarbonSession(storeLocation)
+ .getOrCreateCarbonSession(storeLocation, metastoredb)
spark.sparkContext.setLogLevel("WARN")
spark
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/09fb83ef/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamExample.scala
deleted file mode 100644
index 5ef9d2a..0000000
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/StreamExample.scala
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.examples
-
-import java.io.{File, PrintWriter}
-import java.net.ServerSocket
-
-import org.apache.spark.sql.{CarbonEnv, SparkSession}
-import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
-
-// scalastyle:off println
-object StreamExample {
- def main(args: Array[String]) {
-
- // setup paths
- val rootPath = new File(this.getClass.getResource("/").getPath
- + "../../../..").getCanonicalPath
- val storeLocation = s"$rootPath/examples/spark2/target/store"
- val warehouse = s"$rootPath/examples/spark2/target/warehouse"
- val metastoredb = s"$rootPath/examples/spark2/target"
- val streamTableName = s"stream_table"
-
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
-
- import org.apache.spark.sql.CarbonSession._
- val spark = SparkSession
- .builder()
- .master("local")
- .appName("StreamExample")
- .config("spark.sql.warehouse.dir", warehouse)
- .getOrCreateCarbonSession(storeLocation, metastoredb)
-
- spark.sparkContext.setLogLevel("ERROR")
-
- val requireCreateTable = true
- val useComplexDataType = false
-
- if (requireCreateTable) {
- // drop table if exists previously
- spark.sql(s"DROP TABLE IF EXISTS ${ streamTableName }")
- // Create target carbon table and populate with initial data
- if (useComplexDataType) {
- spark.sql(
- s"""
- | CREATE TABLE ${ streamTableName }(
- | id INT,
- | name STRING,
- | city STRING,
- | salary FLOAT,
- | file struct<school:array<string>, age:int>
- | )
- | STORED BY 'carbondata'
- | TBLPROPERTIES(
- | 'streaming'='true', 'sort_columns'='name', 'dictionary_include'='city')
- | """.stripMargin)
- } else {
- spark.sql(
- s"""
- | CREATE TABLE ${ streamTableName }(
- | id INT,
- | name STRING,
- | city STRING,
- | salary FLOAT
- | )
- | STORED BY 'carbondata'
- | TBLPROPERTIES(
- | 'streaming'='true', 'sort_columns'='name')
- | """.stripMargin)
- }
-
- val carbonTable = CarbonEnv.getCarbonTable(Some("default"), streamTableName)(spark)
- val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier)
- // batch load
- val path = s"$rootPath/examples/spark2/src/main/resources/streamSample.csv"
- spark.sql(
- s"""
- | LOAD DATA LOCAL INPATH '$path'
- | INTO TABLE $streamTableName
- | OPTIONS('HEADER'='true')
- """.stripMargin)
-
- // streaming ingest
- val serverSocket = new ServerSocket(7071)
- val thread1 = startStreaming(spark, tablePath)
- val thread2 = writeSocket(serverSocket)
- val thread3 = showTableCount(spark, streamTableName)
-
- System.out.println("type enter to interrupt streaming")
- System.in.read()
- thread1.interrupt()
- thread2.interrupt()
- thread3.interrupt()
- serverSocket.close()
- }
-
- spark.sql(s"select count(*) from ${ streamTableName }").show(100, truncate = false)
-
- spark.sql(s"select * from ${ streamTableName }").show(100, truncate = false)
-
- // record(id = 100000001) comes from batch segment_0
- // record(id = 1) comes from stream segment_1
- spark.sql(s"select * " +
- s"from ${ streamTableName } " +
- s"where id = 100000001 or id = 1 limit 100").show(100, truncate = false)
-
- // not filter
- spark.sql(s"select * " +
- s"from ${ streamTableName } " +
- s"where id < 10 limit 100").show(100, truncate = false)
-
- if (useComplexDataType) {
- // complex
- spark.sql(s"select file.age, file.school " +
- s"from ${ streamTableName } " +
- s"where where file.age = 30 ").show(100, truncate = false)
- }
-
- spark.stop()
- System.out.println("streaming finished")
- }
-
- def showTableCount(spark: SparkSession, tableName: String): Thread = {
- val thread = new Thread() {
- override def run(): Unit = {
- for (_ <- 0 to 1000) {
- spark.sql(s"select count(*) from $tableName").show(truncate = false)
- Thread.sleep(1000 * 3)
- }
- }
- }
- thread.start()
- thread
- }
-
- def startStreaming(spark: SparkSession, tablePath: CarbonTablePath): Thread = {
- val thread = new Thread() {
- override def run(): Unit = {
- var qry: StreamingQuery = null
- try {
- val readSocketDF = spark.readStream
- .format("socket")
- .option("host", "localhost")
- .option("port", 7071)
- .load()
-
- // Write data from socket stream to carbondata file
- qry = readSocketDF.writeStream
- .format("carbondata")
- .trigger(ProcessingTime("5 seconds"))
- .option("checkpointLocation", tablePath.getStreamingCheckpointDir)
- .option("dbName", "default")
- .option("tableName", "stream_table")
- .start()
-
- qry.awaitTermination()
- } catch {
- case ex: Exception =>
- ex.printStackTrace()
- println("Done reading and writing streaming data")
- } finally {
- qry.stop()
- }
- }
- }
- thread.start()
- thread
- }
-
- def writeSocket(serverSocket: ServerSocket): Thread = {
- val thread = new Thread() {
- override def run(): Unit = {
- // wait for client to connection request and accept
- val clientSocket = serverSocket.accept()
- val socketWriter = new PrintWriter(clientSocket.getOutputStream())
- var index = 0
- for (_ <- 1 to 1000) {
- // write 5 records per iteration
- for (_ <- 0 to 1000) {
- index = index + 1
- socketWriter.println(index.toString + ",name_" + index
- + ",city_" + index + "," + (index * 10000.00).toString +
- ",school_" + index + ":school_" + index + index + "$" + index)
- }
- socketWriter.flush()
- Thread.sleep(1000)
- }
- socketWriter.close()
- System.out.println("Socket closed")
- }
- }
- thread.start()
- thread
- }
-}
-// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/carbondata/blob/09fb83ef/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
index 4dbdfb0..7b823ac 100644
--- a/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
+++ b/streaming/src/main/java/org/apache/carbondata/streaming/segment/StreamSegment.java
@@ -361,7 +361,7 @@ public class StreamSegment {
}
/**
- * update carbonindex file after after a stream batch.
+ * update carbonindex file after a stream batch.
*/
public static void updateIndexFile(String segmentDir) throws IOException {
FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
[15/50] [abbrv] carbondata git commit: [CARBONDATA-2071] Added block
size to BblockletDataMap while initialising
Posted by ja...@apache.org.
[CARBONDATA-2071] Added block size to BblockletDataMap while initialising
This closes #1851
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/44ffaf57
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/44ffaf57
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/44ffaf57
Branch: refs/heads/carbonstore
Commit: 44ffaf57ef5dfbe6f73241d1a1e31536c0ae90d3
Parents: 315f41c
Author: ravipesala <ra...@gmail.com>
Authored: Tue Jan 23 18:31:54 2018 +0530
Committer: QiangCai <qi...@qq.com>
Committed: Wed Jan 24 16:18:26 2018 +0800
----------------------------------------------------------------------
.../core/indexstore/BlockMetaInfo.java | 47 ++++++++++++++++++++
.../indexstore/BlockletDataMapIndexStore.java | 19 ++++----
.../core/indexstore/BlockletDetailInfo.java | 13 ++++++
.../blockletindex/BlockletDataMap.java | 33 +++++++++-----
.../blockletindex/BlockletDataMapModel.java | 12 ++---
.../hadoop/CarbonMultiBlockSplit.java | 29 ++++++++----
.../streaming/CarbonStreamInputFormatTest.java | 2 +-
.../spark/rdd/CarbonIUDMergerRDD.scala | 2 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +-
.../spark/rdd/CarbonScanPartitionRDD.scala | 2 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 14 +++---
11 files changed, 132 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/core/src/main/java/org/apache/carbondata/core/indexstore/BlockMetaInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockMetaInfo.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockMetaInfo.java
new file mode 100644
index 0000000..6a691d5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockMetaInfo.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+/**
+ * Holds the metadata info of the block.
+ */
+public class BlockMetaInfo {
+
+ /**
+ * HDFS locations of a block
+ */
+ private String[] locationInfo;
+
+ /**
+ * Size of block
+ */
+ private long size;
+
+ public BlockMetaInfo(String[] locationInfo, long size) {
+ this.locationInfo = locationInfo;
+ this.size = size;
+ }
+
+ public String[] getLocationInfo() {
+ return locationInfo;
+ }
+
+ public long getSize() {
+ return size;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index 8eae974..ad80fd7 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -78,7 +78,7 @@ public class BlockletDataMapIndexStore
String segmentPath = CarbonTablePath.getSegmentPath(
identifier.getAbsoluteTableIdentifier().getTablePath(),
identifier.getSegmentId());
- Map<String, String[]> locationMap = new HashMap<>();
+ Map<String, BlockMetaInfo> blockMetaInfoMap = new HashMap<>();
CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath);
CarbonFile[] carbonFiles = carbonFile.locationAwareListFiles();
SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
@@ -86,9 +86,11 @@ public class BlockletDataMapIndexStore
PartitionMapFileStore partitionFileStore = new PartitionMapFileStore();
partitionFileStore.readAllPartitionsOfSegment(carbonFiles, segmentPath);
for (CarbonFile file : carbonFiles) {
- locationMap.put(file.getAbsolutePath(), file.getLocations());
+ blockMetaInfoMap
+ .put(file.getAbsolutePath(), new BlockMetaInfo(file.getLocations(), file.getSize()));
}
- dataMap = loadAndGetDataMap(identifier, indexFileStore, partitionFileStore, locationMap);
+ dataMap =
+ loadAndGetDataMap(identifier, indexFileStore, partitionFileStore, blockMetaInfoMap);
} catch (MemoryException e) {
LOGGER.error("memory exception when loading datamap: " + e.getMessage());
throw new RuntimeException(e.getMessage(), e);
@@ -116,7 +118,7 @@ public class BlockletDataMapIndexStore
if (missedIdentifiers.size() > 0) {
Map<String, SegmentIndexFileStore> segmentIndexFileStoreMap = new HashMap<>();
Map<String, PartitionMapFileStore> partitionFileStoreMap = new HashMap<>();
- Map<String, String[]> locationMap = new HashMap<>();
+ Map<String, BlockMetaInfo> blockMetaInfoMap = new HashMap<>();
for (TableBlockIndexUniqueIdentifier identifier: missedIdentifiers) {
SegmentIndexFileStore indexFileStore =
@@ -136,11 +138,12 @@ public class BlockletDataMapIndexStore
partitionFileStore.readAllPartitionsOfSegment(carbonFiles, segmentPath);
partitionFileStoreMap.put(identifier.getSegmentId(), partitionFileStore);
for (CarbonFile file : carbonFiles) {
- locationMap.put(file.getAbsolutePath(), file.getLocations());
+ blockMetaInfoMap.put(file.getAbsolutePath(),
+ new BlockMetaInfo(file.getLocations(), file.getSize()));
}
}
blockletDataMaps.add(
- loadAndGetDataMap(identifier, indexFileStore, partitionFileStore, locationMap));
+ loadAndGetDataMap(identifier, indexFileStore, partitionFileStore, blockMetaInfoMap));
}
}
} catch (Throwable e) {
@@ -192,7 +195,7 @@ public class BlockletDataMapIndexStore
TableBlockIndexUniqueIdentifier identifier,
SegmentIndexFileStore indexFileStore,
PartitionMapFileStore partitionFileStore,
- Map<String, String[]> locationMap)
+ Map<String, BlockMetaInfo> blockMetaInfoMap)
throws IOException, MemoryException {
String uniqueTableSegmentIdentifier =
identifier.getUniqueTableSegmentIdentifier();
@@ -206,7 +209,7 @@ public class BlockletDataMapIndexStore
dataMap.init(new BlockletDataMapModel(identifier.getFilePath(),
indexFileStore.getFileData(identifier.getCarbonIndexFileName()),
partitionFileStore.getPartitions(identifier.getCarbonIndexFileName()),
- partitionFileStore.isPartionedSegment(), locationMap));
+ partitionFileStore.isPartionedSegment(), blockMetaInfoMap));
lruCache.put(identifier.getUniqueTableSegmentIdentifier(), dataMap,
dataMap.getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
index 5f4224c..ce05fe2 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
@@ -56,6 +56,8 @@ public class BlockletDetailInfo implements Serializable, Writable {
private byte[] columnSchemaBinary;
+ private long blockSize;
+
public int getRowCount() {
return rowCount;
}
@@ -104,6 +106,14 @@ public class BlockletDetailInfo implements Serializable, Writable {
this.schemaUpdatedTimeStamp = schemaUpdatedTimeStamp;
}
+ public long getBlockSize() {
+ return blockSize;
+ }
+
+ public void setBlockSize(long blockSize) {
+ this.blockSize = blockSize;
+ }
+
@Override public void write(DataOutput out) throws IOException {
out.writeInt(rowCount);
out.writeShort(pagesCount);
@@ -121,6 +131,7 @@ public class BlockletDetailInfo implements Serializable, Writable {
out.writeLong(blockFooterOffset);
out.writeInt(columnSchemaBinary.length);
out.write(columnSchemaBinary);
+ out.writeLong(blockSize);
}
@Override public void readFields(DataInput in) throws IOException {
@@ -142,6 +153,7 @@ public class BlockletDetailInfo implements Serializable, Writable {
byte[] schemaArray = new byte[bytesSize];
in.readFully(schemaArray);
readColumnSchema(schemaArray);
+ blockSize = in.readLong();
}
/**
@@ -177,6 +189,7 @@ public class BlockletDetailInfo implements Serializable, Writable {
detailInfo.blockletInfo = blockletInfo;
detailInfo.blockFooterOffset = blockFooterOffset;
detailInfo.columnSchemas = columnSchemas;
+ detailInfo.blockSize = blockSize;
return detailInfo;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 7b2c016..b097c66 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -40,6 +40,7 @@ import org.apache.carbondata.core.datamap.dev.DataMapModel;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.indexstore.BlockMetaInfo;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
@@ -103,6 +104,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
private static int BLOCKLET_ID_INDEX = 11;
+ private static int BLOCK_LENGTH = 12;
+
private static int TASK_MIN_VALUES_INDEX = 0;
private static int TASK_MAX_VALUES_INDEX = 1;
@@ -146,18 +149,19 @@ public class BlockletDataMap implements DataMap, Cacheable {
createSummarySchema(segmentProperties, blockletDataMapInfo.getPartitions(), schemaBinary);
}
TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
- String[] locations = blockletDataMapInfo.getLocationMap().get(blockInfo.getFilePath());
+ BlockMetaInfo blockMetaInfo =
+ blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());
// Here it loads info about all blocklets of index
// Only add if the file exists physically. There are scenarios which index file exists inside
// merge index but related carbondata files are deleted. In that case we first check whether
// the file exists physically or not
- if (locations != null) {
+ if (blockMetaInfo != null) {
if (fileFooter.getBlockletList() == null) {
// This is old store scenario, here blocklet information is not available in index file so
// load only block info
summaryRow =
loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
- locations);
+ blockMetaInfo);
} else {
// blocklet ID will start from 0 again only when part file path is changed
if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
@@ -166,7 +170,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
summaryRow =
loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
- locations, relativeBlockletId);
+ blockMetaInfo, relativeBlockletId);
// this is done because relative blocklet id need to be incremented based on the
// total number of blocklets
relativeBlockletId += fileFooter.getBlockletList().size();
@@ -190,7 +194,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter,
SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
- String[] locations, int relativeBlockletId) {
+ BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
int[] minMaxLen = segmentProperties.getColumnsValueSize();
List<BlockletInfo> blockletList = fileFooter.getBlockletList();
CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
@@ -249,10 +253,12 @@ public class BlockletDataMap implements DataMap, Cacheable {
row.setByteArray(serializedData, ordinal++);
// Add block footer offset, it is used if we need to read footer of block
row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
- setLocations(locations, row, ordinal);
+ setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
ordinal++;
// for relative blockelt id i.e blocklet id that belongs to a particular part file
- row.setShort((short) relativeBlockletId++, ordinal);
+ row.setShort((short) relativeBlockletId++, ordinal++);
+ // Store block size
+ row.setLong(blockMetaInfo.getSize(), ordinal);
unsafeMemoryDMStore.addIndexRowToUnsafe(row);
} catch (Exception e) {
throw new RuntimeException(e);
@@ -276,7 +282,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
*/
private DataMapRowImpl loadToUnsafeBlock(DataFileFooter fileFooter,
SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
- String[] locations) {
+ BlockMetaInfo blockMetaInfo) {
int[] minMaxLen = segmentProperties.getColumnsValueSize();
BlockletIndex blockletIndex = fileFooter.getBlockletIndex();
CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
@@ -327,12 +333,15 @@ public class BlockletDataMap implements DataMap, Cacheable {
row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
try {
- setLocations(locations, row, ordinal);
+ setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
ordinal++;
// for relative blocklet id. Value is -1 because in case of old store blocklet info will
// not be present in the index file and in that case we will not knwo the total number of
// blocklets
- row.setShort((short) -1, ordinal);
+ row.setShort((short) -1, ordinal++);
+
+ // store block size
+ row.setLong(blockMetaInfo.getSize(), ordinal);
unsafeMemoryDMStore.addIndexRowToUnsafe(row);
} catch (Exception e) {
throw new RuntimeException(e);
@@ -535,6 +544,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
// for relative blocklet id i.e. blocklet id that belongs to a particular part file.
indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
+ // for storing block length.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
+
unsafeMemoryDMStore =
new UnsafeMemoryDMStore(indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]));
}
@@ -780,6 +792,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
blocklet.setDetailInfo(detailInfo);
detailInfo.setBlockFooterOffset(row.getLong(BLOCK_FOOTER_OFFSET));
detailInfo.setColumnSchemaBinary(getColumnSchemaBinary());
+ detailInfo.setBlockSize(row.getLong(BLOCK_LENGTH));
return blocklet;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
index 85293a1..b3a7f8c 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
@@ -20,6 +20,7 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.indexstore.BlockMetaInfo;
/**
* It is the model object to keep the information to build or initialize BlockletDataMap.
@@ -32,15 +33,16 @@ public class BlockletDataMapModel extends DataMapModel {
private boolean partitionedSegment;
- private Map<String, String[]> locationMap;
+ Map<String, BlockMetaInfo> blockMetaInfoMap;
public BlockletDataMapModel(String filePath, byte[] fileData, List<String> partitions,
- boolean partitionedSegment, Map<String, String[]> locationMap) {
+ boolean partitionedSegment,
+ Map<String, BlockMetaInfo> blockMetaInfoMap) {
super(filePath);
this.fileData = fileData;
this.partitions = partitions;
this.partitionedSegment = partitionedSegment;
- this.locationMap = locationMap;
+ this.blockMetaInfoMap = blockMetaInfoMap;
}
public byte[] getFileData() {
@@ -55,7 +57,7 @@ public class BlockletDataMapModel extends DataMapModel {
return partitionedSegment;
}
- public Map<String, String[]> getLocationMap() {
- return locationMap;
+ public Map<String, BlockMetaInfo> getBlockMetaInfoMap() {
+ return blockMetaInfoMap;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
index aed3449..7c06b60 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
@@ -21,9 +21,10 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.statusmanager.FileFormat;
import org.apache.hadoop.io.Writable;
@@ -55,15 +56,15 @@ public class CarbonMultiBlockSplit extends InputSplit implements Writable {
length = 0;
}
- public CarbonMultiBlockSplit(AbsoluteTableIdentifier identifier, List<CarbonInputSplit> splitList,
- String[] locations) throws IOException {
+ public CarbonMultiBlockSplit(List<CarbonInputSplit> splitList,
+ String[] locations) {
this.splitList = splitList;
this.locations = locations;
calculateLength();
}
- public CarbonMultiBlockSplit(AbsoluteTableIdentifier identifier, List<CarbonInputSplit> splitList,
- String[] locations, FileFormat fileFormat) throws IOException {
+ public CarbonMultiBlockSplit(List<CarbonInputSplit> splitList,
+ String[] locations, FileFormat fileFormat) {
this.splitList = splitList;
this.locations = locations;
this.fileFormat = fileFormat;
@@ -79,7 +80,7 @@ public class CarbonMultiBlockSplit extends InputSplit implements Writable {
}
@Override
- public long getLength() throws IOException, InterruptedException {
+ public long getLength() {
return length;
}
@@ -89,14 +90,24 @@ public class CarbonMultiBlockSplit extends InputSplit implements Writable {
private void calculateLength() {
long total = 0;
- for (CarbonInputSplit split : splitList) {
- total += split.getLength();
+ if (splitList.size() > 0 && splitList.get(0).getDetailInfo() != null) {
+ Map<String, Long> blockSizes = new HashMap<>();
+ for (CarbonInputSplit split : splitList) {
+ blockSizes.put(split.getBlockPath(), split.getDetailInfo().getBlockSize());
+ }
+ for (Map.Entry<String, Long> entry : blockSizes.entrySet()) {
+ total += entry.getValue();
+ }
+ } else {
+ for (CarbonInputSplit split : splitList) {
+ total += split.getLength();
+ }
}
length = total;
}
@Override
- public String[] getLocations() throws IOException, InterruptedException {
+ public String[] getLocations() {
return locations;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java
index d7f9ac2..57f488f 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormatTest.java
@@ -74,7 +74,7 @@ public class CarbonStreamInputFormatTest extends TestCase {
CarbonInputSplit carbonInputSplit = new CarbonInputSplit();
List<CarbonInputSplit> splitList = new ArrayList<>();
splitList.add(carbonInputSplit);
- return new CarbonMultiBlockSplit(identifier, splitList, new String[] { "localhost" },
+ return new CarbonMultiBlockSplit(splitList, new String[] { "localhost" },
FileFormat.ROW_V1);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
index 26fa037..e8180cd 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
@@ -94,7 +94,7 @@ class CarbonIUDMergerRDD[K, V](
val locations = validSplits.head.getLocations
i += 1
new CarbonSparkPartition(id, i,
- new CarbonMultiBlockSplit(absoluteTableIdentifier, validSplits.asJava, locations))
+ new CarbonMultiBlockSplit(validSplits.asJava, locations))
} else {
null
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 48907cb..8d7b044 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -433,7 +433,7 @@ class CarbonMergerRDD[K, V](
if (blockletCount != 0) {
val taskInfo = splitInfo.asInstanceOf[CarbonInputSplitTaskInfo]
- val multiBlockSplit = new CarbonMultiBlockSplit(absoluteTableIdentifier,
+ val multiBlockSplit = new CarbonMultiBlockSplit(
taskInfo.getCarbonInputSplitList,
Array(nodeName))
if (isPartitionTable) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanPartitionRDD.scala
index 1a8943b..5647427 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanPartitionRDD.scala
@@ -115,7 +115,7 @@ class CarbonScanPartitionRDD(alterPartitionModel: AlterPartitionModel,
val splits = blocksPerTask.asScala.map(_.asInstanceOf[CarbonInputSplit])
if (blocksPerTask.size() != 0) {
val multiBlockSplit =
- new CarbonMultiBlockSplit(absoluteTableIdentifier, splits.asJava, Array(node))
+ new CarbonMultiBlockSplit(splits.asJava, Array(node))
val partition = new CarbonSparkPartition(id, partition_num, multiBlockSplit)
result.add(partition)
partition_num += 1
http://git-wip-us.apache.org/repos/asf/carbondata/blob/44ffaf57/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index a04e9e1..f2c3060 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -111,7 +111,7 @@ class CarbonScanRDD(
val streamPartitions: mutable.Buffer[Partition] =
streamSplits.zipWithIndex.map { splitWithIndex =>
val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier,
+ new CarbonMultiBlockSplit(
Seq(splitWithIndex._1.asInstanceOf[CarbonInputSplit]).asJava,
splitWithIndex._1.getLocations,
FileFormat.ROW_V1)
@@ -156,7 +156,7 @@ class CarbonScanRDD(
(0 until bucketedTable.getNumberOfBuckets).map { bucketId =>
val bucketPartitions = bucketed.getOrElse(bucketId.toString, Nil)
val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier,
+ new CarbonMultiBlockSplit(
bucketPartitions.asJava,
bucketPartitions.flatMap(_.getLocations).toArray)
val partition = new CarbonSparkPartition(id, i, multiBlockSplit)
@@ -186,7 +186,7 @@ class CarbonScanRDD(
val splits = blocksPerTask.asScala.map(_.asInstanceOf[CarbonInputSplit])
if (blocksPerTask.size() != 0) {
val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier, splits.asJava, Array(node))
+ new CarbonMultiBlockSplit(splits.asJava, Array(node))
val partition = new CarbonSparkPartition(id, i, multiBlockSplit)
result.add(partition)
i += 1
@@ -200,7 +200,7 @@ class CarbonScanRDD(
// Randomize the blocklets for better shuffling
Random.shuffle(splits.asScala).zipWithIndex.foreach { splitWithIndex =>
val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier,
+ new CarbonMultiBlockSplit(
Seq(splitWithIndex._1.asInstanceOf[CarbonInputSplit]).asJava,
splitWithIndex._1.getLocations)
val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
@@ -215,7 +215,7 @@ class CarbonScanRDD(
.map(_.asInstanceOf[CarbonInputSplit])
.groupBy(f => f.getBlockPath)
.map { blockSplitEntry =>
- new CarbonMultiBlockSplit(identifier,
+ new CarbonMultiBlockSplit(
blockSplitEntry._2.asJava,
blockSplitEntry._2.flatMap(f => f.getLocations).distinct.toArray)
}.toArray.sortBy(_.getLength)(implicitly[Ordering[Long]].reverse)
@@ -257,7 +257,7 @@ class CarbonScanRDD(
f.getSegmentId.concat(f.getBlockPath)
}.values.zipWithIndex.foreach { splitWithIndex =>
val multiBlockSplit =
- new CarbonMultiBlockSplit(identifier,
+ new CarbonMultiBlockSplit(
splitWithIndex._1.asJava,
splitWithIndex._1.flatMap(f => f.getLocations).distinct.toArray)
val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
@@ -306,7 +306,7 @@ class CarbonScanRDD(
.map(_._1)
.toArray
- val multiBlockSplit = new CarbonMultiBlockSplit(null, carbonInputSplits.asJava, locations)
+ val multiBlockSplit = new CarbonMultiBlockSplit(carbonInputSplits.asJava, locations)
new CarbonSparkPartition(id, partitionId, multiBlockSplit)
}
[10/50] [abbrv] carbondata git commit: [CARBONDATA-2046]agg Query
failed when non supported aggregate is present in Query
Posted by ja...@apache.org.
[CARBONDATA-2046]agg Query failed when non supported aggregate is present in Query
Root Cause :- isValidPlan variable was getting overwritten by CarbonReflectionUtils.hasPredicateSubquery(expression) .
Solution :- CarbonReflectionUtils.hasPredicateSubquery(expression) method should be called when isValidPlan is true to avoid overwritten.
This closes #1824
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/23bc051b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/23bc051b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/23bc051b
Branch: refs/heads/carbonstore
Commit: 23bc051b84c2d1f16147ba8f0df79bfc83c543f8
Parents: bef6af3
Author: BJangir <ba...@gmail.com>
Authored: Wed Jan 17 22:32:10 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Jan 19 19:09:38 2018 +0530
----------------------------------------------------------------------
.../TestPreAggregateTableSelection.scala | 14 +++++
.../sql/hive/CarbonPreAggregateRules.scala | 64 ++++++++++++++------
2 files changed, 60 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/23bc051b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 3fefbaf..17d95ef 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -285,6 +285,20 @@ test("test PreAggregate table selection with timeseries and normal together") {
preAggTableValidator(df.queryExecution.analyzed, "maintabletime_agg1_year")
}
+
+ test("test table selection when unsupported aggregate function is present") {
+ sql("drop table if exists maintabletime")
+ sql(
+ "create table maintabletime(year int,month int,name string,salary int,dob string) stored" +
+ " by 'carbondata' tblproperties('sort_scope'='Global_sort','table_blocksize'='23'," +
+ "'sort_columns'='month,year,name')")
+ sql("insert into maintabletime select 10,11,'x',12,'2014-01-01 00:00:00'")
+ sql(
+ "create datamap agg0 on table maintabletime using 'preaggregate' as select name,sum(salary) from " +
+ "maintabletime group by name")
+
+ sql("select var_samp(name) from maintabletime where name='Mikka' ")
+ }
override def afterAll: Unit = {
sql("drop table if exists mainTable")
sql("drop table if exists lineitem")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/23bc051b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index 5db4895..d629c2a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -263,7 +263,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
case agg@Aggregate(
grExp,
aggExp,
- CarbonSubqueryAlias(_, child@CarbonSubqueryAlias(_, l: LogicalRelation)))
+ CarbonSubqueryAlias(alias1, child@CarbonSubqueryAlias(alias2, l: LogicalRelation)))
if l.relation.isInstanceOf[CarbonDatasourceHadoopRelation] &&
l.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonRelation.
metaData.hasAggregateDataMapSchema =>
@@ -295,7 +295,12 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
agg)
Aggregate(updatedGroupExp,
updatedAggExp,
- newChild)
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession,
+ Some(alias1),
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession, Some(alias2), newChild, None),
+ None))
} else {
agg
}
@@ -306,7 +311,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
case agg@Aggregate(
grExp,
aggExp,
- child@CarbonSubqueryAlias(_, l: LogicalRelation))
+ child@CarbonSubqueryAlias(alias, l: LogicalRelation))
if l.relation.isInstanceOf[CarbonDatasourceHadoopRelation] &&
l.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonRelation.
metaData.hasAggregateDataMapSchema =>
@@ -338,7 +343,8 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
agg)
Aggregate(updatedGroupExp,
updatedAggExp,
- newChild)
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession, Some(alias), newChild, None))
} else {
agg
}
@@ -349,7 +355,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
case agg@Aggregate(
grExp,
aggExp,
- Filter(expression, child@CarbonSubqueryAlias(_, l: LogicalRelation)))
+ Filter(expression, child@CarbonSubqueryAlias(alias, l: LogicalRelation)))
if l.relation.isInstanceOf[CarbonDatasourceHadoopRelation] &&
l.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonRelation.
metaData.hasAggregateDataMapSchema =>
@@ -362,17 +368,21 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
carbonTable,
list,
aggregateExpressions)
+
+ if (isValidPlan) {
+ isValidPlan = !CarbonReflectionUtils.hasPredicateSubquery(expression)
+ }
+
// getting the columns from filter expression
- isValidPlan = !CarbonReflectionUtils.hasPredicateSubquery(expression)
if (isValidPlan) {
extractColumnFromExpression(expression, list, carbonTable, true)
}
- if(isValidPlan) {
+ if (isValidPlan) {
val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
aggregateExpressions,
carbonTable,
agg)
- if(null != aggDataMapSchema && null!= childPlan) {
+ if (null != aggDataMapSchema && null != childPlan) {
val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
val (updatedGroupExp, updatedAggExp, newChild, updatedFilterExpression) =
getUpdatedExpressions(grExp,
@@ -387,7 +397,8 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
Aggregate(updatedGroupExp,
updatedAggExp,
Filter(updatedFilterExpression.get,
- newChild))
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession, Some(alias), newChild, None)))
} else {
agg
}
@@ -395,33 +406,44 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
agg
}
case agg@Aggregate(
- grExp,
- aggExp,
- Filter(expression, CarbonSubqueryAlias(_, child@CarbonSubqueryAlias(_, l: LogicalRelation))))
+ grExp,
+ aggExp,
+ Filter(
+ expression,
+ CarbonSubqueryAlias(alias1, child@CarbonSubqueryAlias(alias2, l: LogicalRelation))))
if l.relation.isInstanceOf[CarbonDatasourceHadoopRelation] &&
l.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonRelation.
metaData.hasAggregateDataMapSchema =>
val carbonTable = getCarbonTable(l)
val list = scala.collection.mutable.HashSet.empty[QueryColumn]
val aggregateExpressions = scala.collection.mutable.HashSet.empty[AggregateExpression]
- val isValidPlan = extractQueryColumnsFromAggExpression(
+ var isValidPlan = extractQueryColumnsFromAggExpression(
grExp,
aggExp,
carbonTable,
list,
aggregateExpressions)
- if(isValidPlan) {
+
+ if (isValidPlan) {
+ isValidPlan = !CarbonReflectionUtils.hasPredicateSubquery(expression)
+ }
+
+ // getting the columns from filter expression
+ if (isValidPlan) {
+ extractColumnFromExpression(expression, list, carbonTable, true)
+ }
+ if (isValidPlan) {
val (aggDataMapSchema, childPlan) = getChildDataMapForTransformation(list,
aggregateExpressions,
carbonTable,
agg)
- if(null != aggDataMapSchema && null!= childPlan) {
+ if (null != aggDataMapSchema && null != childPlan) {
val attributes = childPlan.output.asInstanceOf[Seq[AttributeReference]]
- val (updatedGroupExp, updatedAggExp, newChild, None) =
+ val (updatedGroupExp, updatedAggExp, newChild, updatedFilterExpression) =
getUpdatedExpressions(grExp,
aggExp,
child,
- None,
+ Some(expression),
aggDataMapSchema,
attributes,
childPlan,
@@ -429,7 +451,13 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
agg)
Aggregate(updatedGroupExp,
updatedAggExp,
- newChild)
+ Filter(updatedFilterExpression.get,
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession,
+ Some(alias1),
+ CarbonReflectionUtils
+ .getSubqueryAlias(sparkSession, Some(alias2), newChild, None),
+ None)))
} else {
agg
}
[39/50] [abbrv] carbondata git commit: [CARBONDATA-1224] Added page
level reader instead of reading whole blocklet in V3
Posted by ja...@apache.org.
[CARBONDATA-1224] Added page level reader instead of reading whole blocklet in V3
Problem:In V3 format we read the whole blocklet at once to memory in order save IO time. But it turns out to be costlier in case of parallel reading of more carbondata files.For example if we need to compact 50 segments then compactor need to open the readers on all the 50 segments to do merge sort. But the memory consumption is too high if each reader reads whole blocklet to the memory and there is high chances of going out of memory.Solution:In this type of scenarios we can introduce new readers for V3 to read the data page by page instead of reading whole blocklet at once to reduce the memory footprint.
This closes #1089
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d509f17f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d509f17f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d509f17f
Branch: refs/heads/carbonstore
Commit: d509f17fbbf31b4baef23821f700bcbbfc987001
Parents: 181c280
Author: ravipesala <ra...@gmail.com>
Authored: Mon Jan 22 23:17:26 2018 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Tue Jan 30 22:12:09 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 11 ++
.../carbondata/core/datastore/FileHolder.java | 14 +-
.../datastore/chunk/AbstractRawColumnChunk.java | 6 +-
.../chunk/impl/DimensionRawColumnChunk.java | 17 +-
.../chunk/impl/MeasureRawColumnChunk.java | 18 +-
.../chunk/reader/CarbonDataReaderFactory.java | 22 ++-
...mpressedDimensionChunkFileBasedReaderV1.java | 2 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 2 +-
...essedDimChunkFileBasedPageLevelReaderV3.java | 168 +++++++++++++++++++
...mpressedDimensionChunkFileBasedReaderV3.java | 36 ++--
...CompressedMeasureChunkFileBasedReaderV1.java | 2 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 2 +-
...CompressedMeasureChunkFileBasedReaderV3.java | 39 ++---
...essedMsrChunkFileBasedPageLevelReaderV3.java | 153 +++++++++++++++++
.../core/datastore/impl/DFSFileHolderImpl.java | 16 +-
.../core/datastore/impl/FileHolderImpl.java | 19 +--
.../impl/btree/BlockletBTreeLeafNode.java | 54 +++++-
.../BlockletDataRefNodeWrapper.java | 38 +++--
.../executor/impl/AbstractQueryExecutor.java | 1 +
.../scan/executor/infos/BlockExecutionInfo.java | 13 ++
.../carbondata/core/scan/model/QueryModel.java | 13 ++
.../processor/AbstractDataBlockIterator.java | 25 ++-
.../core/scan/result/AbstractScannedResult.java | 69 +++++++-
.../AbstractDetailQueryResultIterator.java | 1 +
.../scan/scanner/AbstractBlockletScanner.java | 38 +++--
.../core/scan/scanner/impl/FilterScanner.java | 3 +-
.../apache/carbondata/core/util/CarbonUtil.java | 17 ++
.../merger/CarbonCompactionExecutor.java | 19 +++
28 files changed, 682 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index cf95dd9..77e8db8 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1556,6 +1556,17 @@ public final class CarbonCommonConstants {
// the maximum characters that can be supported should be less than Short max value
public static final int MAX_CHARS_PER_COLUMN_DEFAULT = 32000;
+ /**
+ * Enabling page level reader for compaction reduces the memory usage while compacting more
+ * number of segments. It allows reading only page by page instead of reaing whole blocklet to
+ * memory.
+ */
+ @CarbonProperty
+ public static final String CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION =
+ "carbon.enable.page.level.reader.in.compaction";
+
+ public static final String CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION_DEFAULT = "true";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
index 712e116..b4130a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.core.datastore;
-import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -103,6 +102,17 @@ public interface FileHolder {
String getQueryId();
- DataInputStream getDataInputStream(String filePath, long offset) throws IOException;
+ /**
+ * Set the flag to read data page by page instead of whole blocklet.
+ *
+ * @param isReadPageByPage
+ */
+ void setReadPageByPage(boolean isReadPageByPage);
+
+ /**
+ * Whether read the data page by page from carbondata file instead of reading whole
+ * blocklet to memory. It is need in case of memory constraint operations.
+ */
+ boolean isReadPageByPage();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java
index d1362c2..05ac9ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/AbstractRawColumnChunk.java
@@ -39,13 +39,13 @@ public abstract class AbstractRawColumnChunk {
protected int columnIndex;
- private int offSet;
+ private long offSet;
protected int length;
private DataChunk3 dataChunkV3;
- public AbstractRawColumnChunk(int columnIndex, ByteBuffer rawData, int offSet, int length) {
+ public AbstractRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length) {
this.columnIndex = columnIndex;
this.rawData = rawData;
this.offSet = offSet;
@@ -102,7 +102,7 @@ public abstract class AbstractRawColumnChunk {
return columnIndex;
}
- public int getOffSet() {
+ public long getOffSet() {
return offSet;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
index cb112c1..705c13c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
@@ -39,7 +39,7 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
private FileHolder fileHolder;
- public DimensionRawColumnChunk(int columnIndex, ByteBuffer rawData, int offSet, int length,
+ public DimensionRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
DimensionColumnChunkReader columnChunkReader) {
super(columnIndex, rawData, offSet, length);
this.chunkReader = columnChunkReader;
@@ -86,6 +86,21 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
return dataChunks[index];
}
+ /**
+ * Convert raw data with specified page number processed to DimensionColumnDataChunk
+ *
+ * @param index
+ * @return
+ */
+ public DimensionColumnDataChunk convertToDimColDataChunkWithOutCache(int index) {
+ assert index < pagesCount;
+ try {
+ return chunkReader.convertToDimensionChunk(this, index);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Override public void freeMemory() {
if (null != dataChunks) {
for (int i = 0; i < dataChunks.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
index d41cf09..fa0777b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
@@ -39,7 +39,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
private FileHolder fileReader;
- public MeasureRawColumnChunk(int columnIndex, ByteBuffer rawData, int offSet, int length,
+ public MeasureRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
MeasureColumnChunkReader chunkReader) {
super(columnIndex, rawData, offSet, length);
this.chunkReader = chunkReader;
@@ -85,6 +85,22 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
return columnPages[columnIndex];
}
+ /**
+ * Convert raw data with specified page number processed to MeasureColumnDataChunk
+ *
+ * @param index
+ * @return
+ */
+ public ColumnPage convertToColumnPageWithOutCache(int index) {
+ assert index < pagesCount;
+
+ try {
+ return chunkReader.convertToColumnPage(this, index);
+ } catch (IOException | MemoryException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Override public void freeMemory() {
if (null != columnPages) {
for (int i = 0; i < columnPages.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
index 8fee760..07cd7b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
@@ -18,10 +18,12 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.v2.CompressedDimensionChunkFileBasedReaderV2;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimChunkFileBasedPageLevelReaderV3;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.v3.CompressedDimensionChunkFileBasedReaderV3;
import org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1;
import org.apache.carbondata.core.datastore.chunk.reader.measure.v2.CompressedMeasureChunkFileBasedReaderV2;
import org.apache.carbondata.core.datastore.chunk.reader.measure.v3.CompressedMeasureChunkFileBasedReaderV3;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.v3.CompressedMsrChunkFileBasedPageLevelReaderV3;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -62,7 +64,8 @@ public class CarbonDataReaderFactory {
* @return dimension column data reader based on version number
*/
public DimensionColumnChunkReader getDimensionColumnChunkReader(ColumnarFormatVersion version,
- BlockletInfo blockletInfo, int[] eachColumnValueSize, String filePath) {
+ BlockletInfo blockletInfo, int[] eachColumnValueSize, String filePath,
+ boolean readPagebyPage) {
switch (version) {
case V1:
return new CompressedDimensionChunkFileBasedReaderV1(blockletInfo, eachColumnValueSize,
@@ -72,8 +75,13 @@ public class CarbonDataReaderFactory {
filePath);
case V3:
default:
- return new CompressedDimensionChunkFileBasedReaderV3(blockletInfo, eachColumnValueSize,
- filePath);
+ if (readPagebyPage) {
+ return new CompressedDimChunkFileBasedPageLevelReaderV3(blockletInfo, eachColumnValueSize,
+ filePath);
+ } else {
+ return new CompressedDimensionChunkFileBasedReaderV3(blockletInfo, eachColumnValueSize,
+ filePath);
+ }
}
}
@@ -86,7 +94,7 @@ public class CarbonDataReaderFactory {
* @return measure column data reader based on version number
*/
public MeasureColumnChunkReader getMeasureColumnChunkReader(ColumnarFormatVersion version,
- BlockletInfo blockletInfo, String filePath) {
+ BlockletInfo blockletInfo, String filePath, boolean readPagebyPage) {
switch (version) {
case V1:
return new CompressedMeasureChunkFileBasedReaderV1(blockletInfo, filePath);
@@ -94,7 +102,11 @@ public class CarbonDataReaderFactory {
return new CompressedMeasureChunkFileBasedReaderV2(blockletInfo, filePath);
case V3:
default:
- return new CompressedMeasureChunkFileBasedReaderV3(blockletInfo, filePath);
+ if (readPagebyPage) {
+ return new CompressedMsrChunkFileBasedPageLevelReaderV3(blockletInfo, filePath);
+ } else {
+ return new CompressedMeasureChunkFileBasedReaderV3(blockletInfo, filePath);
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index 3e45082..27a4d89 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -108,7 +108,7 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
FileHolder fileReader = dimensionRawColumnChunk.getFileReader();
ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
- dataPage = COMPRESSOR.unCompressByte(rawData.array(), dimensionRawColumnChunk.getOffSet(),
+ dataPage = COMPRESSOR.unCompressByte(rawData.array(), (int) dimensionRawColumnChunk.getOffSet(),
dimensionRawColumnChunk.getLength());
// if row id block is present then read the row id chunk and uncompress it
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index 0dea099..b43f89c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -122,7 +122,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
int[] invertedIndexesReverse = null;
int[] rlePage = null;
DataChunk2 dimensionColumnChunk = null;
- int copySourcePoint = dimensionRawColumnChunk.getOffSet();
+ int copySourcePoint = (int) dimensionRawColumnChunk.getOffSet();
int blockIndex = dimensionRawColumnChunk.getColumnIndex();
ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
if (dimensionChunksOffset.size() - 1 == blockIndex) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
new file mode 100644
index 0000000..1edfd09
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.reader.dimension.v3;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.format.DataChunk2;
+import org.apache.carbondata.format.DataChunk3;
+import org.apache.carbondata.format.Encoding;
+
+/**
+ * Dimension column V3 Reader class which will be used to read and uncompress
+ * V3 format data. It reads the data in each page at once unlike whole blocklet. It is
+ * used for memory constraint operations like compaction.
+ * data format
+ * Data Format
+ * <FileHeader>
+ * <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
+ * <Column2 Data ChunkV3><Column2<Page1><Page2><Page3><Page4>>
+ * <Column3 Data ChunkV3><Column3<Page1><Page2><Page3><Page4>>
+ * <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
+ * <File Footer>
+ */
+public class CompressedDimChunkFileBasedPageLevelReaderV3
+ extends CompressedDimensionChunkFileBasedReaderV3 {
+
+ /**
+ * end position of last dimension in carbon data file
+ */
+ private long lastDimensionOffsets;
+
+ public CompressedDimChunkFileBasedPageLevelReaderV3(BlockletInfo blockletInfo,
+ int[] eachColumnValueSize, String filePath) {
+ super(blockletInfo, eachColumnValueSize, filePath);
+ lastDimensionOffsets = blockletInfo.getDimensionOffset();
+ }
+
+ /**
+ * Below method will be used to read the dimension column data form carbon data file
+ * Steps for reading
+ * 1. Get the length of the data to be read
+ * 2. Allocate the direct buffer
+ * 3. read the data from file
+ * 4. Get the data chunk object from data read
+ * 5. Create the raw chunk object and fill the details
+ *
+ * @param fileReader reader for reading the column from carbon data file
+ * @param blockletColumnIndex blocklet index of the column in carbon data file
+ * @return dimension raw chunk
+ */
+ public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader,
+ int blockletColumnIndex) throws IOException {
+ // get the current dimension offset
+ long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex);
+ int length = 0;
+ // to calculate the length of the data to be read
+ // column other than last column we can subtract the offset of current column with
+ // next column and get the total length.
+ // but for last column we need to use lastDimensionOffset which is the end position
+ // of the last dimension, we can subtract current dimension offset from lastDimesionOffset
+ if (dimensionChunksOffset.size() - 1 == blockletColumnIndex) {
+ length = (int) (lastDimensionOffsets - currentDimensionOffset);
+ } else {
+ length = (int) (dimensionChunksOffset.get(blockletColumnIndex + 1) - currentDimensionOffset);
+ }
+ ByteBuffer buffer;
+ // read the data from carbon data file
+ synchronized (fileReader) {
+ buffer = fileReader.readByteBuffer(filePath, currentDimensionOffset,
+ dimensionChunksLength.get(blockletColumnIndex));
+ }
+ // get the data chunk which will have all the details about the data pages
+ DataChunk3 dataChunk = CarbonUtil.readDataChunk3(new ByteArrayInputStream(buffer.array()));
+ DimensionRawColumnChunk rawColumnChunk =
+ getDimensionRawColumnChunk(fileReader, blockletColumnIndex, currentDimensionOffset, length,
+ null, dataChunk);
+
+ return rawColumnChunk;
+ }
+
+ /**
+ * Below method will be used to read the multiple dimension column data in group
+ * and divide into dimension raw chunk object
+ * Steps for reading
+ * 1. Get the length of the data to be read
+ * 2. Allocate the direct buffer
+ * 3. read the data from file
+ * 4. Get the data chunk object from file for each column
+ * 5. Create the raw chunk object and fill the details for each column
+ * 6. increment the offset of the data
+ *
+ * @param fileReader reader which will be used to read the dimension columns data from file
+ * @param startBlockletColumnIndex blocklet index of the first dimension column
+ * @param endBlockletColumnIndex blocklet index of the last dimension column
+ * @ DimensionRawColumnChunk array
+ */
+ protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException {
+ // create raw chunk for each dimension column
+ DimensionRawColumnChunk[] dimensionDataChunks =
+ new DimensionRawColumnChunk[endBlockletColumnIndex - startBlockletColumnIndex + 1];
+ int index = 0;
+ for (int i = startBlockletColumnIndex; i <= endBlockletColumnIndex; i++) {
+ dimensionDataChunks[index] = readRawDimensionChunk(fileReader, i);
+ index++;
+ }
+ return dimensionDataChunks;
+ }
+
+ /**
+ * Below method will be used to convert the compressed dimension chunk raw data to actual data
+ *
+ * @param dimensionRawColumnChunk dimension raw chunk
+ * @param pageNumber number
+ * @return DimensionColumnDataChunk
+ */
+ @Override public DimensionColumnDataChunk convertToDimensionChunk(
+ DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber)
+ throws IOException, MemoryException {
+ // data chunk of page
+ DataChunk2 pageMetadata = null;
+ // data chunk of blocklet column
+ DataChunk3 dataChunk3 = dimensionRawColumnChunk.getDataChunkV3();
+
+ pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber);
+ // calculating the start point of data
+ // as buffer can contain multiple column data, start point will be datachunkoffset +
+ // data chunk length + page offset
+ long offset = dimensionRawColumnChunk.getOffSet() + dimensionChunksLength
+ .get(dimensionRawColumnChunk.getColumnIndex()) + dataChunk3.getPage_offset()
+ .get(pageNumber);
+ int length = pageMetadata.data_page_length;
+ if (hasEncoding(pageMetadata.encoders, Encoding.INVERTED_INDEX)) {
+ length += pageMetadata.rowid_page_length;
+ }
+
+ if (hasEncoding(pageMetadata.encoders, Encoding.RLE)) {
+ length += pageMetadata.rle_page_length;
+ }
+ // get the data buffer
+ ByteBuffer rawData = dimensionRawColumnChunk.getFileReader()
+ .readByteBuffer(filePath, offset, length);
+
+ return decodeDimension(dimensionRawColumnChunk, rawData, pageMetadata, 0);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
index bb828a6..566e9b7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
@@ -103,9 +103,15 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
}
// get the data chunk which will have all the details about the data pages
DataChunk3 dataChunk = CarbonUtil.readDataChunk3(buffer, 0, length);
+ return getDimensionRawColumnChunk(fileReader, blockletColumnIndex, 0, length, buffer,
+ dataChunk);
+ }
+
+ protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileHolder fileReader,
+ int blockletColumnIndex, long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) {
// creating a raw chunks instance and filling all the details
DimensionRawColumnChunk rawColumnChunk =
- new DimensionRawColumnChunk(blockletColumnIndex, buffer, 0, length, this);
+ new DimensionRawColumnChunk(blockletColumnIndex, buffer, offset, length, this);
int numberOfPages = dataChunk.getPage_length().size();
byte[][] maxValueOfEachPage = new byte[numberOfPages][];
byte[][] minValueOfEachPage = new byte[numberOfPages][];
@@ -166,29 +172,11 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
int runningLength = 0;
for (int i = startBlockletColumnIndex; i <= endBlockletColumnIndex; i++) {
int currentLength = (int) (dimensionChunksOffset.get(i + 1) - dimensionChunksOffset.get(i));
- dimensionDataChunks[index] =
- new DimensionRawColumnChunk(i, buffer, runningLength, currentLength, this);
DataChunk3 dataChunk =
CarbonUtil.readDataChunk3(buffer, runningLength, dimensionChunksLength.get(i));
- int numberOfPages = dataChunk.getPage_length().size();
- byte[][] maxValueOfEachPage = new byte[numberOfPages][];
- byte[][] minValueOfEachPage = new byte[numberOfPages][];
- int[] eachPageLength = new int[numberOfPages];
- for (int j = 0; j < minValueOfEachPage.length; j++) {
- maxValueOfEachPage[j] =
- dataChunk.getData_chunk_list().get(j).getMin_max().getMax_values().get(0).array();
- minValueOfEachPage[j] =
- dataChunk.getData_chunk_list().get(j).getMin_max().getMin_values().get(0).array();
- eachPageLength[j] = dataChunk.getData_chunk_list().get(j).getNumberOfRowsInpage();
- }
- dimensionDataChunks[index].setDataChunkV3(dataChunk);
- dimensionDataChunks[index].setFileHolder(fileReader);
- dimensionDataChunks[index].setPagesCount(dataChunk.getPage_length().size());
- dimensionDataChunks[index].setMaxValues(maxValueOfEachPage);
- dimensionDataChunks[index].setMinValues(minValueOfEachPage);
- dimensionDataChunks[index].setRowCount(eachPageLength);
- dimensionDataChunks[index].setOffsets(ArrayUtils
- .toPrimitive(dataChunk.page_offset.toArray(new Integer[dataChunk.page_offset.size()])));
+ dimensionDataChunks[index] =
+ getDimensionRawColumnChunk(fileReader, i, runningLength, currentLength, buffer,
+ dataChunk);
runningLength += currentLength;
index++;
}
@@ -212,7 +200,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
// calculating the start point of data
// as buffer can contain multiple column data, start point will be datachunkoffset +
// data chunk length + page offset
- int offset = rawColumnPage.getOffSet() + dimensionChunksLength
+ int offset = (int) rawColumnPage.getOffSet() + dimensionChunksLength
.get(rawColumnPage.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber);
// first read the data and uncompressed it
return decodeDimension(rawColumnPage, rawData, pageMetadata, offset);
@@ -240,7 +228,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
return false;
}
- private DimensionColumnDataChunk decodeDimension(DimensionRawColumnChunk rawColumnPage,
+ protected DimensionColumnDataChunk decodeDimension(DimensionRawColumnChunk rawColumnPage,
ByteBuffer pageData, DataChunk2 pageMetadata, int offset)
throws IOException, MemoryException {
if (isEncodedWithMeta(pageMetadata)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index fcfd862..ae55375 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -98,7 +98,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta);
ColumnPage decodedPage = codec.decode(measureRawColumnChunk.getRawData().array(),
- measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength());
+ (int) measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength());
decodedPage.setNullBits(dataChunk.getNullValueIndexForColumn());
return decodedPage;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 001c240..2ddc202 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -110,7 +110,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
int pageNumber) throws IOException, MemoryException {
- int copyPoint = measureRawColumnChunk.getOffSet();
+ int copyPoint = (int) measureRawColumnChunk.getOffSet();
int blockIndex = measureRawColumnChunk.getColumnIndex();
ByteBuffer rawData = measureRawColumnChunk.getRawData();
DataChunk2 measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
index e207c82..4f6987b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -94,9 +94,17 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
// get the data chunk which will have all the details about the data pages
DataChunk3 dataChunk =
CarbonUtil.readDataChunk3(buffer, 0, measureColumnChunkLength.get(columnIndex));
+
+ return getMeasureRawColumnChunk(fileReader, columnIndex, 0, dataLength, buffer,
+ dataChunk);
+ }
+
+ protected MeasureRawColumnChunk getMeasureRawColumnChunk(FileHolder fileReader,
+ int columnIndex, long offset, int dataLength, ByteBuffer buffer,
+ DataChunk3 dataChunk) {
// creating a raw chunks instance and filling all the details
MeasureRawColumnChunk rawColumnChunk =
- new MeasureRawColumnChunk(columnIndex, buffer, 0, dataLength, this);
+ new MeasureRawColumnChunk(columnIndex, buffer, offset, dataLength, this);
int numberOfPages = dataChunk.getPage_length().size();
byte[][] maxValueOfEachPage = new byte[numberOfPages][];
byte[][] minValueOfEachPage = new byte[numberOfPages][];
@@ -158,31 +166,10 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
int currentLength =
(int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i));
- MeasureRawColumnChunk measureRawColumnChunk =
- new MeasureRawColumnChunk(i, buffer, runningLength, currentLength, this);
DataChunk3 dataChunk =
CarbonUtil.readDataChunk3(buffer, runningLength, measureColumnChunkLength.get(i));
-
- int numberOfPages = dataChunk.getPage_length().size();
- byte[][] maxValueOfEachPage = new byte[numberOfPages][];
- byte[][] minValueOfEachPage = new byte[numberOfPages][];
- int[] eachPageLength = new int[numberOfPages];
- for (int j = 0; j < minValueOfEachPage.length; j++) {
- maxValueOfEachPage[j] =
- dataChunk.getData_chunk_list().get(j).getMin_max().getMax_values().get(0).array();
- minValueOfEachPage[j] =
- dataChunk.getData_chunk_list().get(j).getMin_max().getMin_values().get(0).array();
- eachPageLength[j] = dataChunk.getData_chunk_list().get(j).getNumberOfRowsInpage();
- }
- measureRawColumnChunk.setDataChunkV3(dataChunk);
- ;
- measureRawColumnChunk.setFileReader(fileReader);
- measureRawColumnChunk.setPagesCount(dataChunk.getPage_length().size());
- measureRawColumnChunk.setMaxValues(maxValueOfEachPage);
- measureRawColumnChunk.setMinValues(minValueOfEachPage);
- measureRawColumnChunk.setRowCount(eachPageLength);
- measureRawColumnChunk.setOffsets(ArrayUtils
- .toPrimitive(dataChunk.page_offset.toArray(new Integer[dataChunk.page_offset.size()])));
+ MeasureRawColumnChunk measureRawColumnChunk =
+ getMeasureRawColumnChunk(fileReader, i, runningLength, currentLength, buffer, dataChunk);
measureDataChunk[index] = measureRawColumnChunk;
runningLength += currentLength;
index++;
@@ -208,7 +195,7 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
// calculating the start point of data
// as buffer can contain multiple column data, start point will be datachunkoffset +
// data chunk length + page offset
- int offset = rawColumnPage.getOffSet() +
+ int offset = (int) rawColumnPage.getOffSet() +
measureColumnChunkLength.get(rawColumnPage.getColumnIndex()) +
dataChunk3.getPage_offset().get(pageNumber);
ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnPage.getRawData(), offset);
@@ -219,7 +206,7 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
/**
* Decode measure column page with page header and raw data starting from offset
*/
- private ColumnPage decodeMeasure(DataChunk2 pageMetadata, ByteBuffer pageData, int offset)
+ protected ColumnPage decodeMeasure(DataChunk2 pageMetadata, ByteBuffer pageData, int offset)
throws MemoryException, IOException {
List<Encoding> encodings = pageMetadata.getEncoders();
List<ByteBuffer> encoderMetas = pageMetadata.getEncoder_meta();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
new file mode 100644
index 0000000..31ff4c0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.reader.measure.v3;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.format.DataChunk2;
+import org.apache.carbondata.format.DataChunk3;
+
+/**
+ * Measure column V3 Reader class which will be used to read and uncompress
+ * V3 format data
+ * data format
+ * Data Format
+ * <FileHeader>
+ * <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
+ * <Column2 Data ChunkV3><Column2<Page1><Page2><Page3><Page4>>
+ * <Column3 Data ChunkV3><Column3<Page1><Page2><Page3><Page4>>
+ * <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
+ * <File Footer>
+ */
+public class CompressedMsrChunkFileBasedPageLevelReaderV3
+ extends CompressedMeasureChunkFileBasedReaderV3 {
+
+ /**
+ * end position of last measure in carbon data file
+ */
+ private long measureOffsets;
+
+ public CompressedMsrChunkFileBasedPageLevelReaderV3(BlockletInfo blockletInfo, String filePath) {
+ super(blockletInfo, filePath);
+ measureOffsets = blockletInfo.getMeasureOffsets();
+ }
+
+ /**
+ * Below method will be used to read the measure column data form carbon data file
+ * 1. Get the length of the data to be read
+ * 2. Allocate the direct buffer
+ * 3. read the data from file
+ * 4. Get the data chunk object from data read
+ * 5. Create the raw chunk object and fill the details
+ *
+ * @param fileReader reader for reading the column from carbon data file
+ * @param blockletColumnIndex blocklet index of the column in carbon data file
+ * @return measure raw chunk
+ */
+ @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader,
+ int blockletColumnIndex) throws IOException {
+ int dataLength = 0;
+ // to calculate the length of the data to be read
+ // column other than last column we can subtract the offset of current column with
+ // next column and get the total length.
+ // but for last column we need to use lastDimensionOffset which is the end position
+ // of the last dimension, we can subtract current dimension offset from lastDimesionOffset
+ if (measureColumnChunkOffsets.size() - 1 == blockletColumnIndex) {
+ dataLength = (int) (measureOffsets - measureColumnChunkOffsets.get(blockletColumnIndex));
+ } else {
+ dataLength =
+ (int) (measureColumnChunkOffsets.get(blockletColumnIndex + 1) - measureColumnChunkOffsets
+ .get(blockletColumnIndex));
+ }
+ ByteBuffer buffer;
+ // read the data from carbon data file
+ synchronized (fileReader) {
+ buffer = fileReader
+ .readByteBuffer(filePath, measureColumnChunkOffsets.get(blockletColumnIndex),
+ measureColumnChunkLength.get(blockletColumnIndex));
+ }
+ // get the data chunk which will have all the details about the data pages
+ DataChunk3 dataChunk = CarbonUtil.readDataChunk3(new ByteArrayInputStream(buffer.array()));
+ return getMeasureRawColumnChunk(fileReader, blockletColumnIndex,
+ measureColumnChunkOffsets.get(blockletColumnIndex), dataLength, null, dataChunk);
+ }
+
+ /**
+ * Below method will be used to read the multiple measure column data in group
+ * and divide into measure raw chunk object
+ * Steps for reading
+ * 1. Get the length of the data to be read
+ * 2. Allocate the direct buffer
+ * 3. read the data from file
+ * 4. Get the data chunk object from file for each column
+ * 5. Create the raw chunk object and fill the details for each column
+ * 6. increment the offset of the data
+ *
+ * @param fileReader reader which will be used to read the measure columns data from file
+ * @param startColumnBlockletIndex blocklet index of the first measure column
+ * @param endColumnBlockletIndex blocklet index of the last measure column
+ * @return MeasureRawColumnChunk array
+ */
+ protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
+ int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
+ // create raw chunk for each measure column
+ MeasureRawColumnChunk[] measureDataChunk =
+ new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1];
+ int index = 0;
+ for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
+ measureDataChunk[index] = readRawMeasureChunk(fileReader, i);
+ index++;
+ }
+ return measureDataChunk;
+ }
+
+ /**
+ * Below method will be used to convert the compressed measure chunk raw data to actual data
+ *
+ * @param rawColumnPage measure raw chunk
+ * @param pageNumber number
+ * @return DimensionColumnDataChunk
+ */
+ @Override public ColumnPage convertToColumnPage(
+ MeasureRawColumnChunk rawColumnPage, int pageNumber)
+ throws IOException, MemoryException {
+ // data chunk of blocklet column
+ DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3();
+ // data chunk of page
+ DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber);
+ // calculating the start point of data
+ // as buffer can contain multiple column data, start point will be datachunkoffset +
+ // data chunk length + page offset
+ long offset = rawColumnPage.getOffSet() + measureColumnChunkLength
+ .get(rawColumnPage.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber);
+ ByteBuffer buffer = rawColumnPage.getFileReader()
+ .readByteBuffer(filePath, offset, pageMetadata.data_page_length);
+
+ ColumnPage decodedPage = decodeMeasure(pageMetadata, buffer, 0);
+ decodedPage.setNullBits(getNullBitSet(pageMetadata.presence));
+ return decodedPage;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
index 3032ec2..182c8eb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
@@ -16,8 +16,6 @@
*/
package org.apache.carbondata.core.datastore.impl;
-import java.io.BufferedInputStream;
-import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
@@ -39,6 +37,8 @@ public class DFSFileHolderImpl implements FileHolder {
private String queryId;
+ private boolean readPageByPage;
+
public DFSFileHolderImpl() {
this.fileNameAndStreamCache =
@@ -151,12 +151,14 @@ public class DFSFileHolderImpl implements FileHolder {
return queryId;
}
- @Override public DataInputStream getDataInputStream(String filePath, long offset)
- throws IOException {
- FSDataInputStream fsDataInputStream = updateCache(filePath);
- fsDataInputStream.seek(offset);
- return new DataInputStream(new BufferedInputStream(fsDataInputStream, 1 * 1024 * 1024));
+ @Override public void setReadPageByPage(boolean isReadPageByPage) {
+ this.readPageByPage = isReadPageByPage;
}
+
+ @Override public boolean isReadPageByPage() {
+ return readPageByPage;
+ }
+
public Map<String, FSDataInputStream> getFileNameAndStreamCache() {
return fileNameAndStreamCache;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
index c867f75..cc589b7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
@@ -17,8 +17,6 @@
package org.apache.carbondata.core.datastore.impl;
-import java.io.BufferedInputStream;
-import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -38,6 +36,8 @@ public class FileHolderImpl implements FileHolder {
private Map<String, FileChannel> fileNameAndStreamCache;
private String queryId;
+ private boolean readPageByPage;
+
/**
* FileHolderImpl Constructor
* It will create the cache
@@ -214,16 +214,11 @@ public class FileHolderImpl implements FileHolder {
return queryId;
}
- @Override public DataInputStream getDataInputStream(String filePath, long offset)
- throws IOException {
- FileInputStream stream = new FileInputStream(filePath);
- long skipped = stream.skip(offset);
- long toSkip = offset - skipped;
- while (toSkip <= 0) {
- skipped = stream.skip(toSkip);
- toSkip = toSkip - skipped;
- }
+ @Override public void setReadPageByPage(boolean isReadPageByPage) {
+ this.readPageByPage = isReadPageByPage;
+ }
- return new DataInputStream(new BufferedInputStream(stream));
+ @Override public boolean isReadPageByPage() {
+ return readPageByPage;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
index 82c4169..94221ba 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -44,6 +44,16 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
private MeasureColumnChunkReader measureColumnChunkReader;
/**
+ * reader for dimension chunk of page level
+ */
+ private DimensionColumnChunkReader dimensionChunksPageLevelReader;
+
+ /**
+ * reader of measure chunk of page level
+ */
+ private MeasureColumnChunkReader measureColumnChunkPageLevelReader;
+
+ /**
* number of pages in blocklet
*/
private int numberOfPages;
@@ -76,12 +86,28 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
.getDimensionColumnChunkReader(builderInfos.getFooterList().get(0).getVersionId(),
builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex),
builderInfos.getDimensionColumnValueSize(),
- builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath());
+ builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath(),
+ false);
// create a instance of measure column chunk reader
measureColumnChunkReader = CarbonDataReaderFactory.getInstance()
.getMeasureColumnChunkReader(builderInfos.getFooterList().get(0).getVersionId(),
builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex),
- builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath());
+ builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath(),
+ false);
+ // create a instance of dimension chunk
+ dimensionChunksPageLevelReader = CarbonDataReaderFactory.getInstance()
+ .getDimensionColumnChunkReader(builderInfos.getFooterList().get(0).getVersionId(),
+ builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex),
+ builderInfos.getDimensionColumnValueSize(),
+ builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath(),
+ true);
+ // create a instance of measure column chunk reader
+ measureColumnChunkPageLevelReader = CarbonDataReaderFactory.getInstance()
+ .getMeasureColumnChunkReader(builderInfos.getFooterList().get(0).getVersionId(),
+ builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex),
+ builderInfos.getFooterList().get(0).getBlockInfo().getTableBlockInfo().getFilePath(),
+ true);
+
this.nodeNumber = nodeNumber;
this.numberOfPages =
builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex).getNumberOfPages();
@@ -111,7 +137,11 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
*/
@Override public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader,
int[][] blockIndexes) throws IOException {
- return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
+ if (fileReader.isReadPageByPage()) {
+ return dimensionChunksPageLevelReader.readRawDimensionChunks(fileReader, blockIndexes);
+ } else {
+ return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
+ }
}
/**
@@ -123,7 +153,11 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
*/
@Override public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndex)
throws IOException {
- return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndex);
+ if (fileReader.isReadPageByPage()) {
+ return dimensionChunksPageLevelReader.readRawDimensionChunk(fileReader, blockIndex);
+ } else {
+ return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndex);
+ }
}
/**
@@ -135,7 +169,11 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
*/
@Override public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader,
int[][] blockIndexes) throws IOException {
- return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
+ if (fileReader.isReadPageByPage()) {
+ return measureColumnChunkPageLevelReader.readRawMeasureChunks(fileReader, blockIndexes);
+ } else {
+ return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
+ }
}
/**
@@ -147,7 +185,11 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
*/
@Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
throws IOException {
- return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
+ if (fileReader.isReadPageByPage()) {
+ return measureColumnChunkPageLevelReader.readRawMeasureChunk(fileReader, blockIndex);
+ } else {
+ return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
index 1585414..097dd8c 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -117,46 +117,56 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
@Override
public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader, int[][] blockIndexes)
throws IOException {
- DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader();
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
}
@Override
public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndexes)
throws IOException {
- DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader();
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndexes);
}
@Override
public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
throws IOException {
- MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader();
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
}
@Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
throws IOException {
- MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader();
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
}
- private DimensionColumnChunkReader getDimensionColumnChunkReader() throws IOException {
+ private DimensionColumnChunkReader getDimensionColumnChunkReader(FileHolder fileReader) {
ColumnarFormatVersion version =
ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
- return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(
- version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(),
- dimensionLens,
- blockInfos.get(index).getFilePath());
+ if (fileReader.isReadPageByPage()) {
+ return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
+ blockInfos.get(index).getFilePath(), true);
+ } else {
+ return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
+ blockInfos.get(index).getFilePath(), false);
+ }
}
- private MeasureColumnChunkReader getMeasureColumnChunkReader() throws IOException {
+ private MeasureColumnChunkReader getMeasureColumnChunkReader(FileHolder fileReader) {
ColumnarFormatVersion version =
ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
- return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(),
- blockInfos.get(index).getFilePath());
+ if (fileReader.isReadPageByPage()) {
+ return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(),
+ blockInfos.get(index).getFilePath(), true);
+ } else {
+ return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(),
+ blockInfos.get(index).getFilePath(), false);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 694f8ae..c33d5ac 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -315,6 +315,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// total number dimension
blockExecutionInfo
.setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
+ blockExecutionInfo.setPrefetchBlocklet(!queryModel.isReadPageByPage());
blockExecutionInfo
.setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index adb6dc6..65f2838 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -223,6 +223,11 @@ public class BlockExecutionInfo {
*/
private String[] deleteDeltaFilePath;
+ /**
+ * whether to prefetch the blocklet data while scanning
+ */
+ private boolean prefetchBlocklet = true;
+
private Map<String, DeleteDeltaVo> deletedRecordsMap;
public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
return absoluteTableIdentifier;
@@ -658,4 +663,12 @@ public class BlockExecutionInfo {
public void setDeletedRecordsMap(Map<String, DeleteDeltaVo> deletedRecordsMap) {
this.deletedRecordsMap = deletedRecordsMap;
}
+
+ public boolean isPrefetchBlocklet() {
+ return prefetchBlocklet;
+ }
+
+ public void setPrefetchBlocklet(boolean prefetchBlocklet) {
+ this.prefetchBlocklet = prefetchBlocklet;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 5e4872b..655ed5e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -112,6 +112,11 @@ public class QueryModel implements Serializable {
private boolean[] isFilterDimensions;
private boolean[] isFilterMeasures;
+ /**
+ * Read the data from carbondata file page by page instead of whole blocklet.
+ */
+ private boolean readPageByPage;
+
public QueryModel() {
tableBlockInfos = new ArrayList<TableBlockInfo>();
queryDimension = new ArrayList<QueryDimension>();
@@ -417,4 +422,12 @@ public class QueryModel implements Serializable {
public void setIsFilterMeasures(boolean[] isFilterMeasures) {
this.isFilterMeasures = isFilterMeasures;
}
+
+ public boolean isReadPageByPage() {
+ return readPageByPage;
+ }
+
+ public void setReadPageByPage(boolean readPageByPage) {
+ this.readPageByPage = readPageByPage;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
index 798d331..eb5e3f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
@@ -139,15 +139,24 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
private AbstractScannedResult getNextScannedResult() throws Exception {
AbstractScannedResult result = null;
- if (dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get()) {
- if (future == null) {
- future = execute();
+ if (blockExecutionInfo.isPrefetchBlocklet()) {
+ if (dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get()) {
+ if (future == null) {
+ future = execute();
+ }
+ result = future.get();
+ nextBlock.set(false);
+ if (dataBlockIterator.hasNext() || nextRead.get()) {
+ nextBlock.set(true);
+ future = execute();
+ }
}
- result = future.get();
- nextBlock.set(false);
- if (dataBlockIterator.hasNext() || nextRead.get()) {
- nextBlock.set(true);
- future = execute();
+ } else {
+ if (dataBlockIterator.hasNext()) {
+ BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolder();
+ if (blocksChunkHolder != null) {
+ result = blockletScanner.scanBlocklet(blocksChunkHolder);
+ }
}
}
return result;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
index 7cc02ad..b089fad 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
@@ -83,7 +84,12 @@ public abstract class AbstractScannedResult {
/**
* Raw dimension chunks;
*/
- protected DimensionRawColumnChunk[] rawColumnChunks;
+ protected DimensionRawColumnChunk[] dimRawColumnChunks;
+
+ /**
+ * Raw dimension chunks;
+ */
+ protected MeasureRawColumnChunk[] msrRawColumnChunks;
/**
* measure column data chunk
*/
@@ -172,8 +178,12 @@ public abstract class AbstractScannedResult {
this.measureDataChunks = measureDataChunks;
}
- public void setRawColumnChunks(DimensionRawColumnChunk[] rawColumnChunks) {
- this.rawColumnChunks = rawColumnChunks;
+ public void setDimRawColumnChunks(DimensionRawColumnChunk[] dimRawColumnChunks) {
+ this.dimRawColumnChunks = dimRawColumnChunks;
+ }
+
+ public void setMsrRawColumnChunks(MeasureRawColumnChunk[] msrRawColumnChunks) {
+ this.msrRawColumnChunks = msrRawColumnChunks;
}
/**
@@ -269,7 +279,7 @@ public abstract class AbstractScannedResult {
DataOutputStream dataOutput = new DataOutputStream(byteStream);
try {
vectorInfos[i].genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(rawColumnChunks,
+ .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks,
rowMapping == null ? j : rowMapping[pageCounter][j], pageCounter, dataOutput);
Object data = vectorInfos[i].genericQueryType
.getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
@@ -325,11 +335,51 @@ public abstract class AbstractScannedResult {
rowCounter = 0;
currentRow = -1;
pageCounter++;
+ fillDataChunks();
if (null != deletedRecordMap) {
currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
}
}
+ /**
+ * This case is used only in case of compaction, since it does not use filter flow.
+ */
+ public void fillDataChunks() {
+ freeDataChunkMemory();
+ if (pageCounter >= numberOfRows.length) {
+ return;
+ }
+ for (int i = 0; i < dimensionDataChunks.length; i++) {
+ if (dimensionDataChunks[i][pageCounter] == null && dimRawColumnChunks[i] != null) {
+ dimensionDataChunks[i][pageCounter] =
+ dimRawColumnChunks[i].convertToDimColDataChunkWithOutCache(pageCounter);
+ }
+ }
+
+ for (int i = 0; i < measureDataChunks.length; i++) {
+ if (measureDataChunks[i][pageCounter] == null && msrRawColumnChunks[i] != null) {
+ measureDataChunks[i][pageCounter] =
+ msrRawColumnChunks[i].convertToColumnPageWithOutCache(pageCounter);
+ }
+ }
+ }
+
+ // free the memory for the last page chunk
+ private void freeDataChunkMemory() {
+ for (int i = 0; i < dimensionDataChunks.length; i++) {
+ if (pageCounter > 0 && dimensionDataChunks[i][pageCounter - 1] != null) {
+ dimensionDataChunks[i][pageCounter - 1].freeMemory();
+ dimensionDataChunks[i][pageCounter - 1] = null;
+ }
+ }
+ for (int i = 0; i < measureDataChunks.length; i++) {
+ if (pageCounter > 0 && measureDataChunks[i][pageCounter - 1] != null) {
+ measureDataChunks[i][pageCounter - 1].freeMemory();
+ measureDataChunks[i][pageCounter - 1] = null;
+ }
+ }
+ }
+
public int numberOfpages() {
return numberOfRows.length;
}
@@ -451,7 +501,7 @@ public abstract class AbstractScannedResult {
DataOutputStream dataOutput = new DataOutputStream(byteStream);
try {
genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(rawColumnChunks, rowId, pageCounter,
+ .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, rowId, pageCounter,
dataOutput);
complexTypeData[i] = byteStream.toByteArray();
} catch (IOException e) {
@@ -481,6 +531,7 @@ public abstract class AbstractScannedResult {
return true;
} else if (pageCounter < numberOfRows.length) {
pageCounter++;
+ fillDataChunks();
rowCounter = 0;
currentRow = -1;
if (null != deletedRecordMap) {
@@ -520,10 +571,10 @@ public abstract class AbstractScannedResult {
}
}
// free the raw chunks
- if (null != rawColumnChunks) {
- for (int i = 0; i < rawColumnChunks.length; i++) {
- if (null != rawColumnChunks[i]) {
- rawColumnChunks[i].freeMemory();
+ if (null != dimRawColumnChunks) {
+ for (int i = 0; i < dimRawColumnChunks.length; i++) {
+ if (null != dimRawColumnChunks[i]) {
+ dimRawColumnChunks[i].freeMemory();
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 8dd61b1..6172b40 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -108,6 +108,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
this.fileReader = FileFactory.getFileHolder(
FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getTablePath()));
this.fileReader.setQueryId(queryModel.getQueryId());
+ this.fileReader.setReadPageByPage(queryModel.isReadPageByPage());
this.execService = execService;
intialiseInfos();
initQueryStatiticsModel();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
index 19d6f48..bf26ca3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
@@ -77,25 +77,34 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
scannedResult.setBlockletId(
blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
.getDataBlock().blockletId());
+ if (!blockExecutionInfo.isPrefetchBlocklet()) {
+ readBlocklet(blocksChunkHolder);
+ }
DimensionRawColumnChunk[] dimensionRawColumnChunks =
blocksChunkHolder.getDimensionRawDataChunk();
DimensionColumnDataChunk[][] dimensionColumnDataChunks =
- new DimensionColumnDataChunk[dimensionRawColumnChunks.length][];
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (dimensionRawColumnChunks[i] != null) {
- dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
- }
- }
- scannedResult.setDimensionChunks(dimensionColumnDataChunks);
+ new DimensionColumnDataChunk[dimensionRawColumnChunks.length][blocksChunkHolder
+ .getDataBlock().numberOfPages()];
MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getMeasureRawDataChunk();
ColumnPage[][] columnPages =
- new ColumnPage[measureRawColumnChunks.length][];
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (measureRawColumnChunks[i] != null) {
- columnPages[i] = measureRawColumnChunks[i].convertToColumnPage();
+ new ColumnPage[measureRawColumnChunks.length][blocksChunkHolder.getDataBlock()
+ .numberOfPages()];
+ scannedResult.setDimensionChunks(dimensionColumnDataChunks);
+ scannedResult.setMeasureChunks(columnPages);
+ scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
+ if (blockExecutionInfo.isPrefetchBlocklet()) {
+ for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
+ if (dimensionRawColumnChunks[i] != null) {
+ dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
+ }
+ }
+ for (int i = 0; i < measureRawColumnChunks.length; i++) {
+ if (measureRawColumnChunks[i] != null) {
+ columnPages[i] = measureRawColumnChunks[i].convertToColumnPage();
+ }
}
}
- scannedResult.setMeasureChunks(columnPages);
int[] numberOfRows = null;
if (blockExecutionInfo.getAllSelectedDimensionBlocksIndexes().length > 0) {
for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
@@ -112,6 +121,7 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
}
}
}
+
// count(*) case there would not be any dimensions are measures selected.
if (numberOfRows == null) {
numberOfRows = new int[blocksChunkHolder.getDataBlock().numberOfPages()];
@@ -127,7 +137,9 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
}
}
scannedResult.setNumberOfRows(numberOfRows);
- scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
+ if (!blockExecutionInfo.isPrefetchBlocklet()) {
+ scannedResult.fillDataChunks();
+ }
// adding statistics for carbon scan time
QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
.get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index 79f9b49..e77093b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -309,7 +309,8 @@ public class FilterScanner extends AbstractBlockletScanner {
scannedResult.setDimensionChunks(dimensionColumnDataChunks);
scannedResult.setIndexes(indexesGroup);
scannedResult.setMeasureChunks(columnPages);
- scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
scannedResult.setNumberOfRows(rowCount);
// adding statistics for carbon scan time
QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index b1c0c30..5d7a09f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -24,6 +24,7 @@ import java.io.Closeable;
import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.UnsupportedEncodingException;
@@ -1467,6 +1468,22 @@ public final class CarbonUtil {
}, offset, length);
}
+ public static DataChunk3 readDataChunk3(InputStream stream) throws IOException {
+ TBaseCreator creator = new ThriftReader.TBaseCreator() {
+ @Override public TBase create() {
+ return new DataChunk3();
+ }
+ };
+ TProtocol binaryIn = new TCompactProtocol(new TIOStreamTransport(stream));
+ TBase t = creator.create();
+ try {
+ t.read(binaryIn);
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ return (DataChunk3) t;
+ }
+
public static DataChunk2 readDataChunk(ByteBuffer dataChunkBuffer, int offset, int length)
throws IOException {
byte[] data = dataChunkBuffer.array();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d509f17f/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
index 65f70a0..f51ced3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
@@ -45,6 +45,7 @@ import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.result.BatchResult;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
+import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -205,6 +206,7 @@ public class CarbonCompactionExecutor {
model.setForcedDetailRawQuery(true);
model.setFilterExpressionResolverTree(null);
model.setConverter(DataTypeUtil.getDataTypeConverter());
+ model.setReadPageByPage(enablePageLevelReaderForCompaction());
List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
@@ -234,4 +236,21 @@ public class CarbonCompactionExecutor {
return model;
}
+ /**
+ * Whether to enable page level reader for compaction or not.
+ */
+ private boolean enablePageLevelReaderForCompaction() {
+ String enablePageReaderProperty = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION,
+ CarbonCommonConstants.CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION_DEFAULT);
+ boolean enablePageReader;
+ try {
+ enablePageReader = Boolean.parseBoolean(enablePageReaderProperty);
+ } catch (Exception e) {
+ enablePageReader = Boolean.parseBoolean(
+ CarbonCommonConstants.CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION_DEFAULT);
+ }
+ return enablePageReader;
+ }
+
}
[31/50] [abbrv] carbondata git commit: [CARBONDATA-2061] Check for
only valid IN_PROGRESS segments
Posted by ja...@apache.org.
[CARBONDATA-2061] Check for only valid IN_PROGRESS segments
Problem:
During operations like drop, delete segment, compaction, IUD there is a check for the IN_PROGRESS segments of a table. This check is simply checking the tblstatus file for IN_PROGRESS segments.
Solution:
The check should validate the IN_PROGRESS segments and decide on Valid and Invalid IN_PROGRESS segments.
This closes #1844
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/12ccf708
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/12ccf708
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/12ccf708
Branch: refs/heads/carbonstore
Commit: 12ccf708f7b10fd9c7667aa37dc04938e76f36c6
Parents: 241b265
Author: dhatchayani <dh...@gmail.com>
Authored: Mon Jan 22 14:42:07 2018 +0530
Committer: kunal642 <ku...@gmail.com>
Committed: Mon Jan 29 13:50:49 2018 +0530
----------------------------------------------------------------------
.../statusmanager/SegmentStatusManager.java | 61 ++++++++++++++------
.../processing/util/CarbonLoaderUtil.java | 8 ++-
2 files changed, 50 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12ccf708/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index e1fadcf..6af0304 100755
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -292,7 +292,7 @@ public class SegmentStatusManager {
// read existing metadata details in load metadata.
listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath);
if (listOfLoadFolderDetailsArray.length != 0) {
- updateDeletionStatus(loadIds, listOfLoadFolderDetailsArray, invalidLoadIds);
+ updateDeletionStatus(identifier, loadIds, listOfLoadFolderDetailsArray, invalidLoadIds);
if (invalidLoadIds.isEmpty()) {
// All or None , if anything fails then dont write
if (carbonTableStatusLock.lockWithRetries()) {
@@ -376,8 +376,8 @@ public class SegmentStatusManager {
// read existing metadata details in load metadata.
listOfLoadFolderDetailsArray = readLoadMetadata(tableFolderPath);
if (listOfLoadFolderDetailsArray.length != 0) {
- updateDeletionStatus(loadDate, listOfLoadFolderDetailsArray, invalidLoadTimestamps,
- loadStartTime);
+ updateDeletionStatus(identifier, loadDate, listOfLoadFolderDetailsArray,
+ invalidLoadTimestamps, loadStartTime);
if (invalidLoadTimestamps.isEmpty()) {
if (carbonTableStatusLock.lockWithRetries()) {
LOG.info("Table status lock has been successfully acquired.");
@@ -471,8 +471,9 @@ public class SegmentStatusManager {
* @param invalidLoadIds
* @return invalidLoadIds
*/
- private static List<String> updateDeletionStatus(List<String> loadIds,
- LoadMetadataDetails[] listOfLoadFolderDetailsArray, List<String> invalidLoadIds) {
+ private static List<String> updateDeletionStatus(AbsoluteTableIdentifier absoluteTableIdentifier,
+ List<String> loadIds, LoadMetadataDetails[] listOfLoadFolderDetailsArray,
+ List<String> invalidLoadIds) {
SegmentStatus segmentStatus = null;
for (String loadId : loadIds) {
boolean loadFound = false;
@@ -488,12 +489,14 @@ public class SegmentStatusManager {
LOG.error("Cannot delete the Segment which is compacted. Segment is " + loadId);
invalidLoadIds.add(loadId);
return invalidLoadIds;
- } else if (SegmentStatus.INSERT_IN_PROGRESS == segmentStatus) {
+ } else if (SegmentStatus.INSERT_IN_PROGRESS == segmentStatus
+ && checkIfValidLoadInProgress(absoluteTableIdentifier, loadId)) {
// if the segment status is in progress then no need to delete that.
LOG.error("Cannot delete the segment " + loadId + " which is load in progress");
invalidLoadIds.add(loadId);
return invalidLoadIds;
- } else if (SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segmentStatus) {
+ } else if (SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segmentStatus
+ && checkIfValidLoadInProgress(absoluteTableIdentifier, loadId)) {
// if the segment status is overwrite in progress, then no need to delete that.
LOG.error("Cannot delete the segment " + loadId + " which is load overwrite " +
"in progress");
@@ -531,9 +534,9 @@ public class SegmentStatusManager {
* @param invalidLoadTimestamps
* @return invalidLoadTimestamps
*/
- public static List<String> updateDeletionStatus(String loadDate,
- LoadMetadataDetails[] listOfLoadFolderDetailsArray, List<String> invalidLoadTimestamps,
- Long loadStartTime) {
+ public static List<String> updateDeletionStatus(AbsoluteTableIdentifier absoluteTableIdentifier,
+ String loadDate, LoadMetadataDetails[] listOfLoadFolderDetailsArray,
+ List<String> invalidLoadTimestamps, Long loadStartTime) {
// For each load timestamp loop through data and if the
// required load timestamp is found then mark
// the metadata as deleted.
@@ -550,14 +553,19 @@ public class SegmentStatusManager {
} else if (SegmentStatus.STREAMING == segmentStatus) {
LOG.info("Ignoring the segment : " + loadMetadata.getLoadName()
+ "as the segment is streaming in progress.");
- } else if (SegmentStatus.MARKED_FOR_DELETE != segmentStatus &&
- SegmentStatus.INSERT_IN_PROGRESS != segmentStatus &&
- SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS != segmentStatus) {
+ } else if (SegmentStatus.INSERT_IN_PROGRESS == segmentStatus && checkIfValidLoadInProgress(
+ absoluteTableIdentifier, loadMetadata.getLoadName())) {
+ LOG.info("Ignoring the segment : " + loadMetadata.getLoadName()
+ + "as the segment is insert in progress.");
+ } else if (SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == segmentStatus
+ && checkIfValidLoadInProgress(absoluteTableIdentifier, loadMetadata.getLoadName())) {
+ LOG.info("Ignoring the segment : " + loadMetadata.getLoadName()
+ + "as the segment is insert overwrite in progress.");
+ } else if (SegmentStatus.MARKED_FOR_DELETE != segmentStatus) {
loadFound = true;
updateSegmentMetadataDetails(loadMetadata);
- LOG.info("Info: " +
- loadStartTimeString + loadMetadata.getLoadStartTime() +
- " Marked for Delete");
+ LOG.info("Info: " + loadStartTimeString + loadMetadata.getLoadStartTime()
+ + " Marked for Delete");
}
}
}
@@ -700,11 +708,30 @@ public class SegmentStatusManager {
SegmentStatus segmentStatus = loaddetail.getSegmentStatus();
if (segmentStatus == SegmentStatus.INSERT_IN_PROGRESS ||
segmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) {
- loadInProgress = true;
+ loadInProgress =
+ checkIfValidLoadInProgress(carbonTable.getAbsoluteTableIdentifier(),
+ loaddetail.getLoadName());
}
}
}
return loadInProgress;
}
+ /**
+ * This method will check for valid IN_PROGRESS segments.
+ * Tries to acquire a lock on the segment and decide on the stale segments
+ * @param absoluteTableIdentifier
+ *
+ */
+ public static Boolean checkIfValidLoadInProgress(AbsoluteTableIdentifier absoluteTableIdentifier,
+ String loadId) {
+ ICarbonLock segmentLock = CarbonLockFactory.getCarbonLockObj(absoluteTableIdentifier,
+ CarbonTablePath.addSegmentPrefix(loadId) + LockUsage.LOCK);
+ try {
+ return !segmentLock.lockWithRetries(1, 0);
+ } finally {
+ segmentLock.unlock();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12ccf708/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 1991017..fdc2cc3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -199,10 +199,14 @@ public final class CarbonLoaderUtil {
// 2. If load or insert into operation is in progress and insert overwrite operation
// is triggered
for (LoadMetadataDetails entry : listOfLoadFolderDetails) {
- if (entry.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) {
+ if (entry.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS
+ && segmentStatusManager.checkIfValidLoadInProgress(
+ absoluteTableIdentifier, entry.getLoadName())) {
throw new RuntimeException("Already insert overwrite is in progress");
} else if (newMetaEntry.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS
- && entry.getSegmentStatus() == SegmentStatus.INSERT_IN_PROGRESS) {
+ && entry.getSegmentStatus() == SegmentStatus.INSERT_IN_PROGRESS
+ && segmentStatusManager.checkIfValidLoadInProgress(
+ absoluteTableIdentifier, entry.getLoadName())) {
throw new RuntimeException("Already insert into or load is in progress");
}
}
[27/50] [abbrv] carbondata git commit: [CARBONDATA-2069] Restrict
create datamap when load is in progress
Posted by ja...@apache.org.
[CARBONDATA-2069] Restrict create datamap when load is in progress
Problem:
1. Load data into maintable
2. create datamap parallelly
preaggregate table will not have any data while data load is successful for main table. This will make the pre-aggregate table inconsistent
Solution: Restrict creation of pre-aggregate table when load is in progress on main table
This closes #1850
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2f4dbb69
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2f4dbb69
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2f4dbb69
Branch: refs/heads/carbonstore
Commit: 2f4dbb694c512aaaaa56ff1f0ed576dc50ac9f9d
Parents: e409632
Author: kunal642 <ku...@gmail.com>
Authored: Tue Jan 23 18:52:48 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sun Jan 28 13:01:03 2018 +0530
----------------------------------------------------------------------
.../CreatePreAggregateTableCommand.scala | 25 ++++++++++++++++----
1 file changed, 21 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f4dbb69/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index c5340c2..a75a06f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -30,8 +30,10 @@ import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable}
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
+import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
+import org.apache.carbondata.spark.util.DataLoadingUtil
/**
* Below command class will be used to create pre-aggregate table
@@ -180,9 +182,24 @@ case class CreatePreAggregateTableCommand(
// This will be used to check if the parent table has any segments or not. If not then no
// need to fire load for pre-aggregate table. Therefore reading the load details for PARENT
// table.
+ DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false, parentTable)
val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetaDataFilepath)
- .nonEmpty
- if (loadAvailable) {
+ if (loadAvailable.exists(load => load.getSegmentStatus == SegmentStatus.INSERT_IN_PROGRESS ||
+ load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) {
+ throw new UnsupportedOperationException(
+ "Cannot create pre-aggregate table when insert is in progress on main table")
+ } else if (loadAvailable.nonEmpty) {
+ val updatedQuery = if (timeSeriesFunction.isDefined) {
+ val dataMap = parentTable.getTableInfo.getDataMapSchemaList.asScala
+ .filter(p => p.getDataMapName
+ .equalsIgnoreCase(dataMapName)).head
+ .asInstanceOf[AggregationDataMapSchema]
+ PreAggregateUtil.createTimeSeriesSelectQueryFromMain(dataMap.getChildSchema,
+ parentTable.getTableName,
+ parentTable.getDatabaseName)
+ } else {
+ queryString
+ }
// Passing segmentToLoad as * because we want to load all the segments into the
// pre-aggregate table even if the user has set some segments on the parent table.
loadCommand.dataFrame = Some(PreAggregateUtil
[42/50] [abbrv] carbondata git commit:
[CARBONDATA-2075][CARBONDATA-1516] It should throw exception when drop
datamap and the table or datamap not exist
Posted by ja...@apache.org.
[CARBONDATA-2075][CARBONDATA-1516] It should throw exception when drop datamap and the table or datamap not exist
error 1:
It should throw no such datamap exception when drop datamap and the datamap does not exist
not IllegalArgumentException or sys.err
It should throw NoSuchDataMapException that extends MalformedCarbonCommandException
error 2:
fix error: drop datamap should throw exception when carbonTable.get.getTableInfo.getDataMapSchemaList.size() == 0
This closes #1858
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/94011c35
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/94011c35
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/94011c35
Branch: refs/heads/carbonstore
Commit: 94011c35fdeadf48180d9bfafbd9125242d14d3d
Parents: c630b7e
Author: xubo245 <60...@qq.com>
Authored: Thu Jan 25 14:47:34 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Jan 31 08:09:57 2018 +0530
----------------------------------------------------------------------
.../preaggregate/TestPreAggregateDrop.scala | 8 +-
.../timeseries/TestTimeSeriesDropSuite.scala | 108 +++++++++++++++++++
.../testsuite/datamap/TestDataMapCommand.scala | 3 -
.../spark/exception/NoSuchDataMapException.java | 33 ++++++
.../datamap/CarbonDropDataMapCommand.scala | 18 ++--
5 files changed, 156 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
index 147cb6d..1138adf 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
@@ -20,12 +20,12 @@ package org.apache.carbondata.integration.spark.testsuite.preaggregate
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.spark.exception.NoSuchDataMapException
+
class TestPreAggregateDrop extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
sql("drop table if exists maintable")
- sql("drop datamap if exists preagg1 on table maintable")
- sql("drop datamap if exists preagg2 on table maintable")
sql("create table maintable (a string, b string, c string) stored by 'carbondata'")
}
@@ -51,7 +51,7 @@ class TestPreAggregateDrop extends QueryTest with BeforeAndAfterAll {
}
test("drop datamap which is not existed") {
- intercept[RuntimeException] {
+ intercept[NoSuchDataMapException] {
sql("drop datamap newpreagg on table maintable")
}
}
@@ -101,8 +101,6 @@ class TestPreAggregateDrop extends QueryTest with BeforeAndAfterAll {
override def afterAll() {
sql("drop table if exists maintable")
sql("drop table if exists maintable1")
- sql("drop datamap if exists preagg1 on table maintable")
- sql("drop datamap if exists preagg2 on table maintable")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
new file mode 100644
index 0000000..f6d41fb
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.integration.spark.testsuite.timeseries
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+
+class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
+
+ override def beforeAll: Unit = {
+ sql(s"DROP TABLE IF EXISTS mainTable")
+ sql(
+ """
+ | CREATE TABLE mainTable(
+ | dataTime timestamp,
+ | name string,
+ | city string,
+ | age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ }
+
+ test("test timeseries drop datamap 1: drop datamap should throw exception if no datamap") {
+ // DROP DATAMAP DataMapName if the DataMapName not exists
+ checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
+ val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+ }
+ assert(e.getMessage.contains("Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap 2: drop datamap should SUCCESS if haveIF EXISTS") {
+ // DROP DATAMAP DataMapName if the DataMapName not exists
+ checkExistence(sql("show datamap on table mainTable"), false, "agg1_month")
+ try {
+ sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTable")
+ assert(true)
+ } catch {
+ case e: Exception =>
+ println(e)
+ assert(false)
+ }
+ }
+
+ test("test timeseries drop datamap 3: drop datamap should throw proper exception") {
+ sql(
+ """create datamap agg1 on table mainTable
+ |using 'preaggregate'
+ |DMPROPERTIES (
+ | 'timeseries.eventTime'='dataTime',
+ | 'timeseries.hierarchy'='month=1,year=1')
+ |as select dataTime, sum(age) from mainTable
+ |group by dataTime
+ """.stripMargin)
+
+ // Before DROP DATAMAP
+ checkExistence(sql("show datamap on table mainTable"), true, "agg1_month", "agg1_year")
+
+ // DROP DATAMAP DataMapName
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+ checkExistence(sql("show datamap on table mainTable"), false, "agg1_month")
+ val e: Exception = intercept[MalformedCarbonCommandException] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTable")
+ }
+ assert(e.getMessage.contains("Datamap with name agg1_month does not exist under table mainTable"))
+ }
+
+ test("test timeseries drop datamap: drop datamap should throw exception if table not exist") {
+ // DROP DATAMAP DataMapName if the DataMapName not exists and
+ checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
+ val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP agg1_month ON TABLE mainTableNotExist")
+ }
+ assert(e.getMessage.contains(
+ "Dropping datamap agg1_month failed: Table or view 'maintablenotexist' not found "))
+ }
+
+ test("test timeseries drop datamap: should throw exception if table not exist with IF EXISTS") {
+ // DROP DATAMAP DataMapName if the DataMapName not exists
+ // DROP DATAMAP should throw exception if table not exist, even though there is IF EXISTS"
+ checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "agg1_month")
+ val e: Exception = intercept[Exception] {
+ sql(s"DROP DATAMAP IF EXISTS agg1_month ON TABLE mainTableNotExist")
+ }
+ assert(e.getMessage.contains(
+ "Dropping datamap agg1_month failed: Table or view 'maintablenotexist' not found "))
+ }
+
+ override def afterAll: Unit = {
+ sql(s"DROP TABLE IF EXISTS mainTable")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index 0c38239..0860da1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -96,7 +96,6 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
"true")
- sql("drop datamap if exists datamap_hiveMetaStoreTable on table hiveMetaStoreTable")
sql("drop table if exists hiveMetaStoreTable")
sql("create table hiveMetaStoreTable (a string, b string, c string) stored by 'carbondata'")
@@ -121,7 +120,6 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
"true")
- sql("drop datamap if exists datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1")
sql("drop table if exists hiveMetaStoreTable_1")
sql("create table hiveMetaStoreTable_1 (a string, b string, c string) stored by 'carbondata'")
@@ -236,7 +234,6 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
test("test preaggregate load for decimal column for hivemetastore") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
- sql("drop datamap if exists uniqdata_agg on table uniqdata")
sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java
new file mode 100644
index 0000000..959e70d
--- /dev/null
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.exception;
+
+/**
+ * if the dataMap does not exist, carbon should throw NoSuchDataMapException
+ */
+public class NoSuchDataMapException extends MalformedCarbonCommandException {
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ public NoSuchDataMapException(String dataMapName, String tableName) {
+ super("Datamap with name " + dataMapName + " does not exist under table " + tableName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94011c35/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 0ad4457..1fa2494 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.execution.command.AtomicRunnableCommand
import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
import org.apache.spark.sql.execution.command.table.CarbonDropTableCommand
-import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.datamap.DataMapStoreManager
@@ -35,6 +34,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.events._
+import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, NoSuchDataMapException}
/**
* Drops the datamap and any related tables associated with the datamap
@@ -72,8 +72,7 @@ case class CarbonDropDataMapCommand(
Some(CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession))
} catch {
case ex: NoSuchTableException =>
- if (!ifExistsSet) throw ex
- else None
+ throw ex
}
if (carbonTable.isDefined && carbonTable.get.getTableInfo.getDataMapSchemaList.size() > 0) {
val dataMapSchema = carbonTable.get.getTableInfo.getDataMapSchemaList.asScala.zipWithIndex.
@@ -110,15 +109,22 @@ case class CarbonDropDataMapCommand(
sparkSession)
OperationListenerBus.getInstance.fireEvent(dropDataMapPostEvent, operationContext)
} else if (!ifExistsSet) {
- throw new IllegalArgumentException(
- s"Datamap with name $dataMapName does not exist under table $tableName")
+ throw new NoSuchDataMapException(dataMapName, tableName)
+ }
+ } else if ((carbonTable.isDefined &&
+ carbonTable.get.getTableInfo.getDataMapSchemaList.size() == 0)) {
+ if (!ifExistsSet) {
+ throw new NoSuchDataMapException(dataMapName, tableName)
}
}
} catch {
+ case e: NoSuchDataMapException =>
+ throw e
case ex: Exception =>
LOGGER.error(ex, s"Dropping datamap $dataMapName failed")
- sys.error(s"Dropping datamap $dataMapName failed: ${ ex.getMessage }")
+ throw new MalformedCarbonCommandException(
+ s"Dropping datamap $dataMapName failed: ${ex.getMessage}")
}
finally {
if (carbonLocks.nonEmpty) {
[34/50] [abbrv] carbondata git commit: Streaming Ingestion
Posted by ja...@apache.org.
Streaming Ingestion
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b73a221a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b73a221a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b73a221a
Branch: refs/heads/carbonstore
Commit: b73a221a16fcbf0b14ad492e3ffc17902b15d4cd
Parents: ab76347
Author: sgururajshetty <sg...@gmail.com>
Authored: Mon Jan 29 19:34:14 2018 +0530
Committer: sgururajshetty <sg...@gmail.com>
Committed: Mon Jan 29 19:34:14 2018 +0530
----------------------------------------------------------------------
docs/configuration-parameters.md | 2 ++
docs/data-management-on-carbondata.md | 8 ++++++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b73a221a/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index fc10646..5794f4d 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -36,6 +36,8 @@ This section provides the details of all the configurations required for the Car
| carbon.ddl.base.hdfs.url | hdfs://hacluster/opt/data | This property is used to configure the HDFS relative path, the path configured in carbon.ddl.base.hdfs.url will be appended to the HDFS path configured in fs.defaultFS. If this path is configured, then user need not pass the complete path while dataload. For example: If absolute path of the csv file is hdfs://10.18.101.155:54310/data/cnbc/2016/xyz.csv, the path "hdfs://10.18.101.155:54310" will come from property fs.defaultFS and user can configure the /data/cnbc/ as carbon.ddl.base.hdfs.url. Now while dataload user can specify the csv path as /2016/xyz.csv. |
| carbon.badRecords.location | /opt/Carbon/Spark/badrecords | Path where the bad records are stored. |
| carbon.data.file.version | 2 | If this parameter value is set to 1, then CarbonData will support the data load which is in old format(0.x version). If the value is set to 2(1.x onwards version), then CarbonData will support the data load of new format only.|
+| carbon.streaming.auto.handoff.enabled | true | If this parameter value is set to true, auto trigger handoff function will be enabled.|
+| carbon.streaming.segment.max.size | 1024000000 | This parameter defines the maximum size of the streaming segment. Setting this parameter to appropriate value will avoid impacting the streaming ingestion. The value is in bytes.|
## Performance Configuration
This section provides the details of all the configurations required for CarbonData Performance Optimization.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b73a221a/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index a4fb40b..d1415ff 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -110,7 +110,14 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
'COMPACTION_PRESERVE_SEGMENTS'='10',
'ALLOWED_COMPACTION_DAYS'='5')
```
+
+ - **Streaming**
+
+ CarbonData supports streaming ingestion for real-time data. You can create the ‘streaming’ table using the following table properties.
+ ```
+ TBLPROPERTIES (‘streaming’='true’)
+ ```
### Example:
@@ -134,6 +141,7 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
'AUTO_LOAD_MERGE'='true',
'COMPACTION_LEVEL_THRESHOLD'='5,6',
'COMPACTION_PRESERVE_SEGMENTS'='10',
+ ‘streaming’='true’,
'ALLOWED_COMPACTION_DAYS'='5')
```
[23/50] [abbrv] carbondata git commit: updated bad records handling
for its default value
Posted by ja...@apache.org.
updated bad records handling for its default value
updated bad records handling for its default value
This closes #1842
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f62f2ca6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f62f2ca6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f62f2ca6
Branch: refs/heads/carbonstore
Commit: f62f2ca69a32142b214873a7b7d2ff5e3e4baf29
Parents: 09fb83e
Author: vandana <va...@gmail.com>
Authored: Mon Jan 22 12:53:19 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Sun Jan 28 14:25:18 2018 +0800
----------------------------------------------------------------------
docs/data-management-on-carbondata.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f62f2ca6/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index 3a2c1d3..a4fb40b 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -398,11 +398,11 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
```
NOTE:
- * BAD_RECORD_ACTION property can have four type of actions for bad records FORCE, REDIRECT, IGNORE and FAIL.
+ * BAD_RECORDS_ACTION property can have four type of actions for bad records FORCE, REDIRECT, IGNORE and FAIL.
+ * FAIL option is its Default value. If the FAIL option is used, then data loading fails if any bad records are found.
* If the REDIRECT option is used, CarbonData will add all bad records in to a separate CSV file. However, this file must not be used for subsequent data loading because the content may not exactly match the source record. You are advised to cleanse the original source record for further data ingestion. This option is used to remind you which records are bad records.
* If the FORCE option is used, then it auto-corrects the data by storing the bad records as NULL before Loading data.
* If the IGNORE option is used, then bad records are neither loaded nor written to the separate CSV file.
- * IF the FAIL option is used, then data loading fails if any bad records are found.
* In loaded data, if all records are bad records, the BAD_RECORDS_ACTION is invalid and the load operation fails.
* The maximum number of characters per column is 100000. If there are more than 100000 characters in a column, data loading will fail.
[41/50] [abbrv] carbondata git commit: [CARBONDATA-2100] Add query
test case to check result of streaming handoff operation
Posted by ja...@apache.org.
[CARBONDATA-2100] Add query test case to check result of streaming handoff operation
Add query test case to check result of streaming handoff operation
This closes #1887
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c630b7e0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c630b7e0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c630b7e0
Branch: refs/heads/carbonstore
Commit: c630b7e0c9b3b0eb53550b5500e7d58904a9376b
Parents: 9b47961
Author: QiangCai <qi...@qq.com>
Authored: Tue Jan 30 18:43:51 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 09:40:32 2018 +0800
----------------------------------------------------------------------
.../org/apache/spark/carbondata/TestStreamingTableOperation.scala | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c630b7e0/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 3de1391..18e52f6 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -527,8 +527,11 @@ class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
Seq(Row(2 * 100))
)
+ val resultBeforeHandoff = sql("select * from streaming.stream_table_handoff order by id, name").collect()
sql("alter table streaming.stream_table_handoff compact 'streaming'")
Thread.sleep(5000)
+ val resultAfterHandoff = sql("select * from streaming.stream_table_handoff order by id, name").collect()
+ assertResult(resultBeforeHandoff)(resultAfterHandoff)
val newSegments = sql("show segments for table streaming.stream_table_handoff").collect()
assert(newSegments.length == 3 || newSegments.length == 5)
assertResult("Streaming")(newSegments((newSegments.length - 1) / 2).getString(1))
[35/50] [abbrv] carbondata git commit: [CARBONDATA-1616]
Documentation for Streaming Ingestion
Posted by ja...@apache.org.
[CARBONDATA-1616] Documentation for Streaming Ingestion
This closes #1877
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9ec64a6d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9ec64a6d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9ec64a6d
Branch: refs/heads/carbonstore
Commit: 9ec64a6d2b4d06fee7b8bb9db3beb559ff23fc0d
Parents: b73a221
Author: sgururajshetty <sg...@gmail.com>
Authored: Mon Jan 29 19:39:04 2018 +0530
Committer: QiangCai <qi...@qq.com>
Committed: Tue Jan 30 12:03:52 2018 +0800
----------------------------------------------------------------------
docs/data-management-on-carbondata.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ec64a6d/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index d1415ff..3af95ac 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -116,7 +116,7 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
CarbonData supports streaming ingestion for real-time data. You can create the ‘streaming’ table using the following table properties.
```
- TBLPROPERTIES (‘streaming’='true’)
+ TBLPROPERTIES ('streaming'='true')
```
### Example:
@@ -141,7 +141,7 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
'AUTO_LOAD_MERGE'='true',
'COMPACTION_LEVEL_THRESHOLD'='5,6',
'COMPACTION_PRESERVE_SEGMENTS'='10',
- ‘streaming’='true’,
+ 'streaming'='true',
'ALLOWED_COMPACTION_DAYS'='5')
```
[49/50] [abbrv] carbondata git commit: [CARBONDATA-1968] Add external
table support
Posted by ja...@apache.org.
[CARBONDATA-1968] Add external table support
This PR adds support for creating external table with existing carbondata files, using Hive syntax.
CREATE EXTERNAL TABLE tableName STORED BY 'carbondata' LOCATION 'path'
This closes #1749
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d74e6e43
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d74e6e43
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d74e6e43
Branch: refs/heads/carbonstore
Commit: d74e6e4367d80c10cbe8ed58516fd045bb0c1009
Parents: 3b4c51c
Author: Jacky Li <ja...@qq.com>
Authored: Tue Jan 2 23:46:14 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Jan 31 13:17:14 2018 +0800
----------------------------------------------------------------------
.../core/metadata/schema/table/CarbonTable.java | 9 ++
.../createTable/TestCreateExternalTable.scala | 91 ++++++++++++++++++++
.../TestDataWithDicExcludeAndInclude.scala | 10 ---
.../command/table/CarbonDropTableCommand.scala | 5 +-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 64 +++++++++-----
5 files changed, 147 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d74e6e43/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 4bb0d20..07989b2 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -785,6 +785,15 @@ public class CarbonTable implements Serializable {
&& !tableInfo.getParentRelationIdentifiers().isEmpty();
}
+ /**
+ * Return true if this is an external table (table with property "_external"="true", this is
+ * an internal table property set during table creation)
+ */
+ public boolean isExternalTable() {
+ String external = tableInfo.getFactTable().getTableProperties().get("_external");
+ return external != null && external.equalsIgnoreCase("true");
+ }
+
public long size() throws IOException {
Map<String, Long> dataIndexSize = CarbonUtil.calculateDataIndexSize(this);
Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d74e6e43/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
new file mode 100644
index 0000000..67370eb
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.createTable
+
+import java.io.File
+
+import org.apache.spark.sql.{AnalysisException, CarbonEnv}
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
+
+ var originDataPath: String = _
+
+ override def beforeAll(): Unit = {
+ sql("DROP TABLE IF EXISTS origin")
+ // create carbon table and insert data
+ sql("CREATE TABLE origin(key INT, value STRING) STORED BY 'carbondata'")
+ sql("INSERT INTO origin select 100,'spark'")
+ sql("INSERT INTO origin select 200,'hive'")
+ originDataPath = s"$storeLocation/origin"
+ }
+
+ override def afterAll(): Unit = {
+ sql("DROP TABLE IF EXISTS origin")
+ }
+
+ test("create external table with existing files") {
+ assert(new File(originDataPath).exists())
+ sql("DROP TABLE IF EXISTS source")
+
+ // create external table with existing files
+ sql(
+ s"""
+ |CREATE EXTERNAL TABLE source
+ |STORED BY 'carbondata'
+ |LOCATION '$storeLocation/origin'
+ """.stripMargin)
+ checkAnswer(sql("SELECT count(*) from source"), sql("SELECT count(*) from origin"))
+
+ val carbonTable = CarbonEnv.getCarbonTable(None, "source")(sqlContext.sparkSession)
+ assert(carbonTable.isExternalTable)
+
+ sql("DROP TABLE IF EXISTS source")
+
+ // DROP TABLE should not delete data
+ assert(new File(originDataPath).exists())
+ }
+
+ test("create external table with empty folder") {
+ val exception = intercept[AnalysisException] {
+ sql(
+ s"""
+ |CREATE EXTERNAL TABLE source
+ |STORED BY 'carbondata'
+ |LOCATION './nothing'
+ """.stripMargin)
+ }
+ assert(exception.getMessage().contains("Invalid table path provided"))
+ }
+
+ test("create external table with CTAS") {
+ val exception = intercept[AnalysisException] {
+ sql(
+ """
+ |CREATE EXTERNAL TABLE source
+ |STORED BY 'carbondata'
+ |LOCATION './nothing'
+ |AS
+ | SELECT * FROM origin
+ """.stripMargin)
+ }
+ assert(exception.getMessage().contains("Create external table as select"))
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d74e6e43/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
index c788857..201da39 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
@@ -90,16 +90,6 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with BeforeA
)
}
- test("test create external table should fail") {
- assert(intercept[AnalysisException](
- sql(
- """
- | CREATE EXTERNAL TABLE t1 (id string, value int)
- | STORED BY 'carbondata'
- """.stripMargin)
- ).message.contains("Operation not allowed: CREATE EXTERNAL TABLE"))
- }
-
override def afterAll {
dropTable
CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d74e6e43/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
index 9c0eb57..5e00914 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
@@ -144,7 +144,10 @@ case class CarbonDropTableCommand(
// delete the table folder
val tablePath = carbonTable.getTablePath
val fileType = FileFactory.getFileType(tablePath)
- if (FileFactory.isFileExist(tablePath, fileType)) {
+
+ // delete table data only if it is not external table
+ if (FileFactory.isFileExist(tablePath, fileType) &&
+ !carbonTable.isExternalTable) {
val file = FileFactory.getCarbonFile(tablePath, fileType)
CarbonUtil.deleteFoldersAndFilesSilent(file)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d74e6e43/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index 4b77417..ad6d0c7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -32,6 +32,8 @@ import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.CarbonReflectionUtils
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.hadoop.util.SchemaReader
import org.apache.carbondata.spark.CarbonOption
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -164,9 +166,6 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
if (bucketSpecContext != null) {
operationNotAllowed("CREATE TABLE ... CLUSTERED BY", bucketSpecContext)
}
- if (external) {
- operationNotAllowed("CREATE EXTERNAL TABLE", tableHeader)
- }
val cols = Option(columns).toSeq.flatMap(visitColTypeList)
val properties = getPropertyKeyValues(tablePropertyList)
@@ -231,6 +230,10 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
operationNotAllowed(
"Schema may not be specified in a Create Table As Select (CTAS) statement", columns)
}
+ // external table is not allow
+ if (external) {
+ operationNotAllowed("Create external table as select", tableHeader)
+ }
fields = parser
.getFields(CarbonEnv.getInstance(sparkSession).carbonMetastore
.getSchemaFromUnresolvedRelation(sparkSession, Some(q).get))
@@ -242,29 +245,48 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf,
}
// validate tblProperties
val bucketFields = parser.getBucketFields(tableProperties, fields, options)
- // prepare table model of the collected tokens
- val tableModel: TableModel = parser.prepareTableModel(
- ifNotExists,
- convertDbNameToLowerCase(tableIdentifier.database),
- tableIdentifier.table.toLowerCase,
- fields,
- partitionFields,
- tableProperties,
- bucketFields,
- isAlterFlow = false,
- tableComment)
+ val tableInfo = if (external) {
+ // read table info from schema file in the provided table path
+ val identifier = AbsoluteTableIdentifier.from(
+ tablePath.get,
+ CarbonEnv.getDatabaseName(tableIdentifier.database)(sparkSession),
+ tableIdentifier.table)
+ val table = try {
+ SchemaReader.getTableInfo(identifier)
+ } catch {
+ case e: Throwable =>
+ operationNotAllowed(s"Invalid table path provided: ${tablePath.get} ", tableHeader)
+ }
+ // set "_external" property, so that DROP TABLE will not delete the data
+ table.getFactTable.getTableProperties.put("_external", "true")
+ table
+ } else {
+ // prepare table model of the collected tokens
+ val tableModel: TableModel = parser.prepareTableModel(
+ ifNotExists,
+ convertDbNameToLowerCase(tableIdentifier.database),
+ tableIdentifier.table.toLowerCase,
+ fields,
+ partitionFields,
+ tableProperties,
+ bucketFields,
+ isAlterFlow = false,
+ tableComment)
+ TableNewProcessor(tableModel)
+ }
selectQuery match {
case query@Some(q) =>
CarbonCreateTableAsSelectCommand(
- TableNewProcessor(tableModel),
- query.get,
- tableModel.ifNotExistsSet,
- tablePath)
+ tableInfo = tableInfo,
+ query = query.get,
+ ifNotExistsSet = ifNotExists,
+ tableLocation = tablePath)
case _ =>
- CarbonCreateTableCommand(TableNewProcessor(tableModel),
- tableModel.ifNotExistsSet,
- tablePath)
+ CarbonCreateTableCommand(
+ tableInfo = tableInfo,
+ ifNotExistsSet = ifNotExists,
+ tableLocation = tablePath)
}
}
[44/50] [abbrv] carbondata git commit: [CARBONDATA-1987] Make package
name and directory paths consistent;
remove duplicate file CarbonColumnValidator
Posted by ja...@apache.org.
[CARBONDATA-1987] Make package name and directory paths consistent;remove duplicate file CarbonColumnValidator
add coveralls token to spark-2.2 profile;synchronize file path and package name;Delete duplicate class CarbonColumnValidator present in spark2 module
This closes #1764
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4d3f3989
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4d3f3989
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4d3f3989
Branch: refs/heads/carbonstore
Commit: 4d3f3989b5b3aef6ed44e3c67c4102bea4505013
Parents: 94011c3
Author: Raghunandan S <ca...@gmail.com>
Authored: Thu Jan 4 20:18:07 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Jan 31 08:19:22 2018 +0530
----------------------------------------------------------------------
.../spark/CarbonColumnValidator.scala | 36 --
.../src/main/spark2.1/CarbonSQLConf.scala | 149 -------
.../src/main/spark2.1/CarbonSessionState.scala | 339 ----------------
.../apache/spark/sql/hive/CarbonSQLConf.scala | 149 +++++++
.../spark/sql/hive/CarbonSessionState.scala | 339 ++++++++++++++++
.../src/main/spark2.2/CarbonSessionState.scala | 398 -------------------
.../src/main/spark2.2/CarbonSqlConf.scala | 148 -------
.../spark/sql/hive/CarbonSessionState.scala | 398 +++++++++++++++++++
.../apache/spark/sql/hive/CarbonSqlConf.scala | 148 +++++++
pom.xml | 8 +-
10 files changed, 1039 insertions(+), 1073 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
deleted file mode 100644
index 03c4764..0000000
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark
-
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
- /**
- * Carbon column validator
- */
-class CarbonColumnValidator extends ColumnValidator {
- def validateColumns(allColumns: Seq[ColumnSchema]): Unit = {
- allColumns.foreach { columnSchema =>
- val colWithSameId = allColumns.filter { x =>
- x.getColumnUniqueId.equals(columnSchema.getColumnUniqueId)
- }
- if (colWithSameId.size > 1) {
- throw new MalformedCarbonCommandException("Two column can not have same columnId")
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala b/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
deleted file mode 100644
index 15ccb0c..0000000
--- a/integration/spark2/src/main/spark2.1/CarbonSQLConf.scala
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.internal.SQLConf.SQLConfigBuilder
-
-import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * To initialize dynamic values default param
- */
-class CarbonSQLConf(sparkSession: SparkSession) {
-
- val carbonProperties = CarbonProperties.getInstance()
-
- /**
- * To initialize dynamic param defaults along with usage docs
- */
- def addDefaultCarbonParams(): Unit = {
- val ENABLE_UNSAFE_SORT =
- SQLConfigBuilder(CarbonCommonConstants.ENABLE_UNSAFE_SORT)
- .doc("To enable/ disable unsafe sort.")
- .booleanConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
- val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
- SQLConfigBuilder(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
- .doc("To set carbon task distribution.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
- val BAD_RECORDS_LOGGER_ENABLE =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
- .doc("To enable/ disable carbon bad record logger.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants
- .CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
- val BAD_RECORDS_ACTION =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
- .doc("To configure the bad records action.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
- val IS_EMPTY_DATA_BAD_RECORD =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
- .doc("Property to decide weather empty data to be considered bad/ good record.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT
- .toBoolean)
- val SORT_SCOPE =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE)
- .doc("Property to specify sort scope.")
- .stringConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- val BATCH_SORT_SIZE_INMB =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB)
- .doc("Property to specify batch sort size in MB.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
- val SINGLE_PASS =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS)
- .doc("Property to enable/disable single_pass.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
- val BAD_RECORD_PATH =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH)
- .doc("Property to configure the bad record location.")
- .stringConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- val GLOBAL_SORT_PARTITIONS =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS)
- .doc("Property to configure the global sort partitions.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
- CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
- val DATEFORMAT =
- SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
- .doc("Property to configure data format for date type columns.")
- .stringConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
- val CARBON_INPUT_SEGMENTS = SQLConfigBuilder(
- "carbon.input.segments.<database_name>.<table_name>")
- .doc("Property to configure the list of segments to query.").stringConf
- .createWithDefault(carbonProperties
- .getProperty("carbon.input.segments.<database_name>.<table_name>", "*"))
- }
- /**
- * to set the dynamic properties default values
- */
- def addDefaultCarbonSessionParams(): Unit = {
- sparkSession.conf.set(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
- sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
- CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
- CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
deleted file mode 100644
index 0fe0f96..0000000
--- a/integration/spark2/src/main/spark2.1/CarbonSessionState.scala
+++ /dev/null
@@ -1,339 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
-import org.apache.spark.sql.catalyst.catalog.{CatalogTablePartition, FunctionResourceLoader, GlobalTempViewManager, SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BoundReference, Expression, InterpretedPredicate, PredicateSubquery, ScalarSubquery}
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.ParserUtils._
-import org.apache.spark.sql.catalyst.parser.SqlBaseParser.CreateTableContext
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias}
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.catalyst.{CatalystConf, TableIdentifier}
-import org.apache.spark.sql.execution.datasources._
-import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
-import org.apache.spark.sql.execution.{SparkOptimizer, SparkSqlAstBuilder}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
-import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlParser, CarbonSparkSqlParser}
-import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, ExperimentalMethods, SparkSession, Strategy}
-
-import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-/**
- * This class will have carbon catalog and refresh the relation from cache if the carbontable in
- * carbon catalog is not same as cached carbon relation's carbon table
- *
- * @param externalCatalog
- * @param globalTempViewManager
- * @param sparkSession
- * @param functionResourceLoader
- * @param functionRegistry
- * @param conf
- * @param hadoopConf
- */
-class CarbonSessionCatalog(
- externalCatalog: HiveExternalCatalog,
- globalTempViewManager: GlobalTempViewManager,
- sparkSession: SparkSession,
- functionResourceLoader: FunctionResourceLoader,
- functionRegistry: FunctionRegistry,
- conf: SQLConf,
- hadoopConf: Configuration)
- extends HiveSessionCatalog(
- externalCatalog,
- globalTempViewManager,
- sparkSession,
- functionResourceLoader,
- functionRegistry,
- conf,
- hadoopConf) {
-
- lazy val carbonEnv = {
- val env = new CarbonEnv
- env.init(sparkSession)
- env
- }
-
- // Initialize all listeners to the Operation bus.
- CarbonEnv.initListeners()
-
- /**
- * This method will invalidate carbonrelation from cache if carbon table is updated in
- * carbon catalog
- *
- * @param name
- * @param alias
- * @return
- */
- override def lookupRelation(name: TableIdentifier,
- alias: Option[String]): LogicalPlan = {
- val rtnRelation = super.lookupRelation(name, alias)
- var toRefreshRelation = false
- rtnRelation match {
- case SubqueryAlias(_,
- LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _), _) =>
- toRefreshRelation = refreshRelationFromCache(name, alias, carbonDatasourceHadoopRelation)
- case LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _) =>
- toRefreshRelation = refreshRelationFromCache(name, alias, carbonDatasourceHadoopRelation)
- case _ =>
- }
-
- if (toRefreshRelation) {
- super.lookupRelation(name, alias)
- } else {
- rtnRelation
- }
- }
-
- private def refreshRelationFromCache(identifier: TableIdentifier,
- alias: Option[String],
- carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation): Boolean = {
- var isRefreshed = false
- val storePath = CarbonProperties.getStorePath
- carbonEnv.carbonMetastore.
- checkSchemasModifiedTimeAndReloadTable(identifier)
-
- val table = carbonEnv.carbonMetastore.getTableFromMetadataCache(
- carbonDatasourceHadoopRelation.carbonTable.getDatabaseName,
- carbonDatasourceHadoopRelation.carbonTable.getTableName)
- if (table.isEmpty || (table.isDefined &&
- table.get.getTableLastUpdatedTime !=
- carbonDatasourceHadoopRelation.carbonTable.getTableLastUpdatedTime)) {
- refreshTable(identifier)
- DataMapStoreManager.getInstance().
- clearDataMaps(AbsoluteTableIdentifier.from(storePath,
- identifier.database.getOrElse("default"), identifier.table))
- isRefreshed = true
- logInfo(s"Schema changes have been detected for table: $identifier")
- }
- isRefreshed
- }
-
- /**
- * returns hive client from session state
- *
- * @return
- */
- def getClient(): org.apache.spark.sql.hive.client.HiveClient = {
- sparkSession.sessionState.asInstanceOf[CarbonSessionState].metadataHive
- }
-
- override def createPartitions(
- tableName: TableIdentifier,
- parts: Seq[CatalogTablePartition],
- ignoreIfExists: Boolean): Unit = {
- try {
- val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
- // Get the properties from thread local
- val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
- if (carbonSessionInfo != null) {
- val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
- super.createPartitions(tableName, updatedParts, ignoreIfExists)
- } else {
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
- } catch {
- case e: Exception =>
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
- }
-
- /**
- * This is alternate way of getting partition information. It first fetches all partitions from
- * hive and then apply filter instead of querying hive along with filters.
- * @param partitionFilters
- * @param sparkSession
- * @param identifier
- * @return
- */
- def getPartitionsAlternate(
- partitionFilters: Seq[Expression],
- sparkSession: SparkSession,
- identifier: TableIdentifier) = {
- val allPartitions = sparkSession.sessionState.catalog.listPartitions(identifier)
- val catalogTable = sparkSession.sessionState.catalog.getTableMetadata(identifier)
- val partitionSchema = catalogTable.partitionSchema
- if (partitionFilters.nonEmpty) {
- val boundPredicate =
- InterpretedPredicate.create(partitionFilters.reduce(And).transform {
- case att: AttributeReference =>
- val index = partitionSchema.indexWhere(_.name == att.name)
- BoundReference(index, partitionSchema(index).dataType, nullable = true)
- })
- allPartitions.filter { p => boundPredicate(p.toRow(partitionSchema)) }
- } else {
- allPartitions
- }
- }
-}
-
-/**
- * Session state implementation to override sql parser and adding strategies
- * @param sparkSession
- */
-class CarbonSessionState(sparkSession: SparkSession) extends HiveSessionState(sparkSession) {
-
- override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
-
- experimentalMethods.extraStrategies = extraStrategies
-
- experimentalMethods.extraOptimizations = extraOptimizations
-
- def extraStrategies: Seq[Strategy] = {
- Seq(
- new StreamingTableStrategy(sparkSession),
- new CarbonLateDecodeStrategy,
- new DDLStrategy(sparkSession)
- )
- }
-
- def extraOptimizations: Seq[Rule[LogicalPlan]] = {
- Seq(new CarbonIUDRule,
- new CarbonUDFTransformRule,
- new CarbonLateDecodeRule)
- }
-
- override lazy val optimizer: Optimizer = new CarbonOptimizer(catalog, conf, experimentalMethods)
-
- def extendedAnalyzerRules: Seq[Rule[LogicalPlan]] = Nil
- def internalAnalyzerRules: Seq[Rule[LogicalPlan]] = {
- catalog.ParquetConversions ::
- catalog.OrcConversions ::
- CarbonPreInsertionCasts(sparkSession) ::
- CarbonIUDAnalysisRule(sparkSession) ::
- AnalyzeCreateTable(sparkSession) ::
- PreprocessTableInsertion(conf) ::
- DataSourceAnalysis(conf) ::
- (if (conf.runSQLonFile) {
- new ResolveDataSource(sparkSession) :: Nil
- } else { Nil })
- }
-
- override lazy val analyzer: Analyzer =
- new CarbonAnalyzer(catalog, conf, sparkSession,
- new Analyzer(catalog, conf) {
- override val extendedResolutionRules =
- if (extendedAnalyzerRules.nonEmpty) {
- extendedAnalyzerRules ++ internalAnalyzerRules
- } else {
- internalAnalyzerRules
- }
- override val extendedCheckRules = Seq(
- PreWriteCheck(conf, catalog))
- }
- )
-
- /**
- * Internal catalog for managing table and database states.
- */
- override lazy val catalog = {
- new CarbonSessionCatalog(
- sparkSession.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog],
- sparkSession.sharedState.globalTempViewManager,
- sparkSession,
- functionResourceLoader,
- functionRegistry,
- conf,
- newHadoopConf())
- }
-}
-
-class CarbonAnalyzer(catalog: SessionCatalog,
- conf: CatalystConf,
- sparkSession: SparkSession,
- analyzer: Analyzer) extends Analyzer(catalog, conf) {
- override def execute(plan: LogicalPlan): LogicalPlan = {
- var logicalPlan = analyzer.execute(plan)
- logicalPlan = CarbonPreAggregateDataLoadingRules(sparkSession).apply(logicalPlan)
- CarbonPreAggregateQueryRules(sparkSession).apply(logicalPlan)
- }
-}
-
-class CarbonOptimizer(
- catalog: SessionCatalog,
- conf: SQLConf,
- experimentalMethods: ExperimentalMethods)
- extends SparkOptimizer(catalog, conf, experimentalMethods) {
-
- override def execute(plan: LogicalPlan): LogicalPlan = {
- val transFormedPlan: LogicalPlan = CarbonOptimizerUtil.transformForScalarSubQuery(plan)
- super.execute(transFormedPlan)
- }
-}
-
-object CarbonOptimizerUtil {
- def transformForScalarSubQuery(plan: LogicalPlan) : LogicalPlan = {
- // In case scalar subquery add flag in relation to skip the decoder plan in optimizer rule,
- // And optimize whole plan at once.
- val transFormedPlan = plan.transform {
- case filter: Filter =>
- filter.transformExpressions {
- case s: ScalarSubquery =>
- val tPlan = s.plan.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- ScalarSubquery(tPlan, s.children, s.exprId)
- case p: PredicateSubquery =>
- val tPlan = p.plan.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- PredicateSubquery(tPlan, p.children, p.nullAware, p.exprId)
- }
- }
- transFormedPlan
- }
-}
-
-class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
- extends SparkSqlAstBuilder(conf) {
-
- val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
-
- override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = {
- val fileStorage = helper.getFileStorage(ctx.createFileFormat)
-
- if (fileStorage.equalsIgnoreCase("'carbondata'") ||
- fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
- helper.createCarbonTable(
- tableHeader = ctx.createTableHeader,
- skewSpecContext = ctx.skewSpec,
- bucketSpecContext = ctx.bucketSpec,
- partitionColumns = ctx.partitionColumns,
- columns = ctx.columns,
- tablePropertyList = ctx.tablePropertyList,
- locationSpecContext = ctx.locationSpec(),
- tableComment = Option(ctx.STRING()).map(string),
- ctas = ctx.AS,
- query = ctx.query)
- } else {
- super.visitCreateTable(ctx)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSQLConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSQLConf.scala b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSQLConf.scala
new file mode 100644
index 0000000..15ccb0c
--- /dev/null
+++ b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSQLConf.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.internal.SQLConf.SQLConfigBuilder
+
+import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * To initialize dynamic values default param
+ */
+class CarbonSQLConf(sparkSession: SparkSession) {
+
+ val carbonProperties = CarbonProperties.getInstance()
+
+ /**
+ * To initialize dynamic param defaults along with usage docs
+ */
+ def addDefaultCarbonParams(): Unit = {
+ val ENABLE_UNSAFE_SORT =
+ SQLConfigBuilder(CarbonCommonConstants.ENABLE_UNSAFE_SORT)
+ .doc("To enable/ disable unsafe sort.")
+ .booleanConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
+ val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
+ SQLConfigBuilder(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
+ .doc("To set carbon task distribution.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
+ val BAD_RECORDS_LOGGER_ENABLE =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
+ .doc("To enable/ disable carbon bad record logger.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants
+ .CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
+ val BAD_RECORDS_ACTION =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
+ .doc("To configure the bad records action.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
+ val IS_EMPTY_DATA_BAD_RECORD =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
+ .doc("Property to decide weather empty data to be considered bad/ good record.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT
+ .toBoolean)
+ val SORT_SCOPE =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE)
+ .doc("Property to specify sort scope.")
+ .stringConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
+ val BATCH_SORT_SIZE_INMB =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB)
+ .doc("Property to specify batch sort size in MB.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
+ val SINGLE_PASS =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS)
+ .doc("Property to enable/disable single_pass.")
+ .booleanConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
+ val BAD_RECORD_PATH =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH)
+ .doc("Property to configure the bad record location.")
+ .stringConf
+ .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ val GLOBAL_SORT_PARTITIONS =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS)
+ .doc("Property to configure the global sort partitions.")
+ .stringConf
+ .createWithDefault(carbonProperties
+ .getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
+ CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
+ val DATEFORMAT =
+ SQLConfigBuilder(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
+ .doc("Property to configure data format for date type columns.")
+ .stringConf
+ .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
+ val CARBON_INPUT_SEGMENTS = SQLConfigBuilder(
+ "carbon.input.segments.<database_name>.<table_name>")
+ .doc("Property to configure the list of segments to query.").stringConf
+ .createWithDefault(carbonProperties
+ .getProperty("carbon.input.segments.<database_name>.<table_name>", "*"))
+ }
+ /**
+ * to set the dynamic properties default values
+ */
+ def addDefaultCarbonSessionParams(): Unit = {
+ sparkSession.conf.set(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
+ sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
+ carbonProperties
+ .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
+ CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
+ carbonProperties.getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
+ CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
+ sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
new file mode 100644
index 0000000..0fe0f96
--- /dev/null
+++ b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
+import org.apache.spark.sql.catalyst.catalog.{CatalogTablePartition, FunctionResourceLoader, GlobalTempViewManager, SessionCatalog}
+import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BoundReference, Expression, InterpretedPredicate, PredicateSubquery, ScalarSubquery}
+import org.apache.spark.sql.catalyst.optimizer.Optimizer
+import org.apache.spark.sql.catalyst.parser.ParserInterface
+import org.apache.spark.sql.catalyst.parser.ParserUtils._
+import org.apache.spark.sql.catalyst.parser.SqlBaseParser.CreateTableContext
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.{CatalystConf, TableIdentifier}
+import org.apache.spark.sql.execution.datasources._
+import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
+import org.apache.spark.sql.execution.{SparkOptimizer, SparkSqlAstBuilder}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
+import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlParser, CarbonSparkSqlParser}
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, ExperimentalMethods, SparkSession, Strategy}
+
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
+import org.apache.carbondata.spark.util.CarbonScalaUtil
+
+/**
+ * This class will have carbon catalog and refresh the relation from cache if the carbontable in
+ * carbon catalog is not same as cached carbon relation's carbon table
+ *
+ * @param externalCatalog
+ * @param globalTempViewManager
+ * @param sparkSession
+ * @param functionResourceLoader
+ * @param functionRegistry
+ * @param conf
+ * @param hadoopConf
+ */
+class CarbonSessionCatalog(
+ externalCatalog: HiveExternalCatalog,
+ globalTempViewManager: GlobalTempViewManager,
+ sparkSession: SparkSession,
+ functionResourceLoader: FunctionResourceLoader,
+ functionRegistry: FunctionRegistry,
+ conf: SQLConf,
+ hadoopConf: Configuration)
+ extends HiveSessionCatalog(
+ externalCatalog,
+ globalTempViewManager,
+ sparkSession,
+ functionResourceLoader,
+ functionRegistry,
+ conf,
+ hadoopConf) {
+
+ lazy val carbonEnv = {
+ val env = new CarbonEnv
+ env.init(sparkSession)
+ env
+ }
+
+ // Initialize all listeners to the Operation bus.
+ CarbonEnv.initListeners()
+
+ /**
+ * This method will invalidate carbonrelation from cache if carbon table is updated in
+ * carbon catalog
+ *
+ * @param name
+ * @param alias
+ * @return
+ */
+ override def lookupRelation(name: TableIdentifier,
+ alias: Option[String]): LogicalPlan = {
+ val rtnRelation = super.lookupRelation(name, alias)
+ var toRefreshRelation = false
+ rtnRelation match {
+ case SubqueryAlias(_,
+ LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _), _) =>
+ toRefreshRelation = refreshRelationFromCache(name, alias, carbonDatasourceHadoopRelation)
+ case LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _) =>
+ toRefreshRelation = refreshRelationFromCache(name, alias, carbonDatasourceHadoopRelation)
+ case _ =>
+ }
+
+ if (toRefreshRelation) {
+ super.lookupRelation(name, alias)
+ } else {
+ rtnRelation
+ }
+ }
+
+ private def refreshRelationFromCache(identifier: TableIdentifier,
+ alias: Option[String],
+ carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation): Boolean = {
+ var isRefreshed = false
+ val storePath = CarbonProperties.getStorePath
+ carbonEnv.carbonMetastore.
+ checkSchemasModifiedTimeAndReloadTable(identifier)
+
+ val table = carbonEnv.carbonMetastore.getTableFromMetadataCache(
+ carbonDatasourceHadoopRelation.carbonTable.getDatabaseName,
+ carbonDatasourceHadoopRelation.carbonTable.getTableName)
+ if (table.isEmpty || (table.isDefined &&
+ table.get.getTableLastUpdatedTime !=
+ carbonDatasourceHadoopRelation.carbonTable.getTableLastUpdatedTime)) {
+ refreshTable(identifier)
+ DataMapStoreManager.getInstance().
+ clearDataMaps(AbsoluteTableIdentifier.from(storePath,
+ identifier.database.getOrElse("default"), identifier.table))
+ isRefreshed = true
+ logInfo(s"Schema changes have been detected for table: $identifier")
+ }
+ isRefreshed
+ }
+
+ /**
+ * returns hive client from session state
+ *
+ * @return
+ */
+ def getClient(): org.apache.spark.sql.hive.client.HiveClient = {
+ sparkSession.sessionState.asInstanceOf[CarbonSessionState].metadataHive
+ }
+
+ override def createPartitions(
+ tableName: TableIdentifier,
+ parts: Seq[CatalogTablePartition],
+ ignoreIfExists: Boolean): Unit = {
+ try {
+ val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ // Get the properties from thread local
+ val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
+ if (carbonSessionInfo != null) {
+ val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
+ super.createPartitions(tableName, updatedParts, ignoreIfExists)
+ } else {
+ super.createPartitions(tableName, parts, ignoreIfExists)
+ }
+ } catch {
+ case e: Exception =>
+ super.createPartitions(tableName, parts, ignoreIfExists)
+ }
+ }
+
+ /**
+ * This is alternate way of getting partition information. It first fetches all partitions from
+ * hive and then apply filter instead of querying hive along with filters.
+ * @param partitionFilters
+ * @param sparkSession
+ * @param identifier
+ * @return
+ */
+ def getPartitionsAlternate(
+ partitionFilters: Seq[Expression],
+ sparkSession: SparkSession,
+ identifier: TableIdentifier) = {
+ val allPartitions = sparkSession.sessionState.catalog.listPartitions(identifier)
+ val catalogTable = sparkSession.sessionState.catalog.getTableMetadata(identifier)
+ val partitionSchema = catalogTable.partitionSchema
+ if (partitionFilters.nonEmpty) {
+ val boundPredicate =
+ InterpretedPredicate.create(partitionFilters.reduce(And).transform {
+ case att: AttributeReference =>
+ val index = partitionSchema.indexWhere(_.name == att.name)
+ BoundReference(index, partitionSchema(index).dataType, nullable = true)
+ })
+ allPartitions.filter { p => boundPredicate(p.toRow(partitionSchema)) }
+ } else {
+ allPartitions
+ }
+ }
+}
+
+/**
+ * Session state implementation to override sql parser and adding strategies
+ * @param sparkSession
+ */
+class CarbonSessionState(sparkSession: SparkSession) extends HiveSessionState(sparkSession) {
+
+ override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
+
+ experimentalMethods.extraStrategies = extraStrategies
+
+ experimentalMethods.extraOptimizations = extraOptimizations
+
+ def extraStrategies: Seq[Strategy] = {
+ Seq(
+ new StreamingTableStrategy(sparkSession),
+ new CarbonLateDecodeStrategy,
+ new DDLStrategy(sparkSession)
+ )
+ }
+
+ def extraOptimizations: Seq[Rule[LogicalPlan]] = {
+ Seq(new CarbonIUDRule,
+ new CarbonUDFTransformRule,
+ new CarbonLateDecodeRule)
+ }
+
+ override lazy val optimizer: Optimizer = new CarbonOptimizer(catalog, conf, experimentalMethods)
+
+ def extendedAnalyzerRules: Seq[Rule[LogicalPlan]] = Nil
+ def internalAnalyzerRules: Seq[Rule[LogicalPlan]] = {
+ catalog.ParquetConversions ::
+ catalog.OrcConversions ::
+ CarbonPreInsertionCasts(sparkSession) ::
+ CarbonIUDAnalysisRule(sparkSession) ::
+ AnalyzeCreateTable(sparkSession) ::
+ PreprocessTableInsertion(conf) ::
+ DataSourceAnalysis(conf) ::
+ (if (conf.runSQLonFile) {
+ new ResolveDataSource(sparkSession) :: Nil
+ } else { Nil })
+ }
+
+ override lazy val analyzer: Analyzer =
+ new CarbonAnalyzer(catalog, conf, sparkSession,
+ new Analyzer(catalog, conf) {
+ override val extendedResolutionRules =
+ if (extendedAnalyzerRules.nonEmpty) {
+ extendedAnalyzerRules ++ internalAnalyzerRules
+ } else {
+ internalAnalyzerRules
+ }
+ override val extendedCheckRules = Seq(
+ PreWriteCheck(conf, catalog))
+ }
+ )
+
+ /**
+ * Internal catalog for managing table and database states.
+ */
+ override lazy val catalog = {
+ new CarbonSessionCatalog(
+ sparkSession.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog],
+ sparkSession.sharedState.globalTempViewManager,
+ sparkSession,
+ functionResourceLoader,
+ functionRegistry,
+ conf,
+ newHadoopConf())
+ }
+}
+
+class CarbonAnalyzer(catalog: SessionCatalog,
+ conf: CatalystConf,
+ sparkSession: SparkSession,
+ analyzer: Analyzer) extends Analyzer(catalog, conf) {
+ override def execute(plan: LogicalPlan): LogicalPlan = {
+ var logicalPlan = analyzer.execute(plan)
+ logicalPlan = CarbonPreAggregateDataLoadingRules(sparkSession).apply(logicalPlan)
+ CarbonPreAggregateQueryRules(sparkSession).apply(logicalPlan)
+ }
+}
+
+class CarbonOptimizer(
+ catalog: SessionCatalog,
+ conf: SQLConf,
+ experimentalMethods: ExperimentalMethods)
+ extends SparkOptimizer(catalog, conf, experimentalMethods) {
+
+ override def execute(plan: LogicalPlan): LogicalPlan = {
+ val transFormedPlan: LogicalPlan = CarbonOptimizerUtil.transformForScalarSubQuery(plan)
+ super.execute(transFormedPlan)
+ }
+}
+
+object CarbonOptimizerUtil {
+ def transformForScalarSubQuery(plan: LogicalPlan) : LogicalPlan = {
+ // In case scalar subquery add flag in relation to skip the decoder plan in optimizer rule,
+ // And optimize whole plan at once.
+ val transFormedPlan = plan.transform {
+ case filter: Filter =>
+ filter.transformExpressions {
+ case s: ScalarSubquery =>
+ val tPlan = s.plan.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ ScalarSubquery(tPlan, s.children, s.exprId)
+ case p: PredicateSubquery =>
+ val tPlan = p.plan.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ PredicateSubquery(tPlan, p.children, p.nullAware, p.exprId)
+ }
+ }
+ transFormedPlan
+ }
+}
+
+class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
+ extends SparkSqlAstBuilder(conf) {
+
+ val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
+
+ override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = {
+ val fileStorage = helper.getFileStorage(ctx.createFileFormat)
+
+ if (fileStorage.equalsIgnoreCase("'carbondata'") ||
+ fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
+ helper.createCarbonTable(
+ tableHeader = ctx.createTableHeader,
+ skewSpecContext = ctx.skewSpec,
+ bucketSpecContext = ctx.bucketSpec,
+ partitionColumns = ctx.partitionColumns,
+ columns = ctx.columns,
+ tablePropertyList = ctx.tablePropertyList,
+ locationSpecContext = ctx.locationSpec(),
+ tableComment = Option(ctx.STRING()).map(string),
+ ctas = ctx.AS,
+ query = ctx.query)
+ } else {
+ super.visitCreateTable(ctx)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
deleted file mode 100644
index 3c151f0..0000000
--- a/integration/spark2/src/main/spark2.2/CarbonSessionState.scala
+++ /dev/null
@@ -1,398 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-
-import scala.collection.generic.SeqFactory
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
-import org.apache.spark.sql.catalyst.catalog._
-import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BoundReference, Exists, Expression, In, InterpretedPredicate, ListQuery, ScalarSubquery}
-import org.apache.spark.sql.catalyst.optimizer.Optimizer
-import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.parser.ParserUtils.string
-import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{AddTableColumnsContext, ChangeColumnContext, CreateHiveTableContext, CreateTableContext}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias}
-import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.command.schema.{CarbonAlterTableAddColumnCommand, CarbonAlterTableDataTypeChangeCommand}
-import org.apache.spark.sql.execution.datasources.{FindDataSourceTable, LogicalRelation, PreWriteCheck, ResolveSQLOnFile, _}
-import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
-import org.apache.spark.sql.execution.{SparkOptimizer, SparkSqlAstBuilder}
-import org.apache.spark.sql.hive.client.HiveClient
-import org.apache.spark.sql.internal.{SQLConf, SessionState}
-import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
-import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlParser, CarbonSparkSqlParser}
-import org.apache.spark.sql.types.DecimalType
-import org.apache.spark.util.CarbonReflectionUtils
-
-import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-/**
- * This class will have carbon catalog and refresh the relation from cache if the carbontable in
- * carbon catalog is not same as cached carbon relation's carbon table
- *
- * @param externalCatalog
- * @param globalTempViewManager
- * @param sparkSession
- * @param functionResourceLoader
- * @param functionRegistry
- * @param conf
- * @param hadoopConf
- */
-class CarbonSessionCatalog(
- externalCatalog: HiveExternalCatalog,
- globalTempViewManager: GlobalTempViewManager,
- functionRegistry: FunctionRegistry,
- sparkSession: SparkSession,
- conf: SQLConf,
- hadoopConf: Configuration,
- parser: ParserInterface,
- functionResourceLoader: FunctionResourceLoader)
- extends HiveSessionCatalog(
- externalCatalog,
- globalTempViewManager,
- new HiveMetastoreCatalog(sparkSession),
- functionRegistry,
- conf,
- hadoopConf,
- parser,
- functionResourceLoader
- ) {
-
- lazy val carbonEnv = {
- val env = new CarbonEnv
- env.init(sparkSession)
- env
- }
-
- def getCarbonEnv() : CarbonEnv = {
- carbonEnv
- }
-
- // Initialize all listeners to the Operation bus.
- CarbonEnv.initListeners()
-
-
-
-
- override def lookupRelation(name: TableIdentifier): LogicalPlan = {
- val rtnRelation = super.lookupRelation(name)
- var toRefreshRelation = false
- rtnRelation match {
- case SubqueryAlias(_,
- LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
- toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
- case LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _) =>
- toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
- case SubqueryAlias(_, relation) if
- relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
- relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.HiveTableRelation") ||
- relation.getClass.getName.equals(
- "org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation") =>
- val catalogTable =
- CarbonReflectionUtils.getFieldOfCatalogTable(
- "tableMeta",
- relation).asInstanceOf[CatalogTable]
- toRefreshRelation =
- CarbonEnv.refreshRelationFromCache(catalogTable.identifier)(sparkSession)
- case _ =>
- }
-
- if (toRefreshRelation) {
- super.lookupRelation(name)
- } else {
- rtnRelation
- }
- }
-
- /**
- * returns hive client from HiveExternalCatalog
- *
- * @return
- */
- def getClient(): org.apache.spark.sql.hive.client.HiveClient = {
- sparkSession.asInstanceOf[CarbonSession].sharedState.externalCatalog
- .asInstanceOf[HiveExternalCatalog].client
- }
-
- override def createPartitions(
- tableName: TableIdentifier,
- parts: Seq[CatalogTablePartition],
- ignoreIfExists: Boolean): Unit = {
- try {
- val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
- // Get the properties from thread local
- val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
- if (carbonSessionInfo != null) {
- val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
- super.createPartitions(tableName, updatedParts, ignoreIfExists)
- } else {
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
- } catch {
- case e: Exception =>
- super.createPartitions(tableName, parts, ignoreIfExists)
- }
- }
-
- /**
- * This is alternate way of getting partition information. It first fetches all partitions from
- * hive and then apply filter instead of querying hive along with filters.
- * @param partitionFilters
- * @param sparkSession
- * @param identifier
- * @return
- */
- def getPartitionsAlternate(partitionFilters: Seq[Expression],
- sparkSession: SparkSession,
- identifier: TableIdentifier) = {
- val allPartitions = sparkSession.sessionState.catalog.listPartitions(identifier)
- ExternalCatalogUtils.prunePartitionsByFilter(
- sparkSession.sessionState.catalog.getTableMetadata(identifier),
- allPartitions,
- partitionFilters,
- sparkSession.sessionState.conf.sessionLocalTimeZone)
- }
-}
-
-
-class CarbonAnalyzer(catalog: SessionCatalog,
- conf: SQLConf,
- sparkSession: SparkSession,
- analyzer: Analyzer) extends Analyzer(catalog, conf) {
- override def execute(plan: LogicalPlan): LogicalPlan = {
- var logicalPlan = analyzer.execute(plan)
- logicalPlan = CarbonPreAggregateDataLoadingRules(sparkSession).apply(logicalPlan)
- CarbonPreAggregateQueryRules(sparkSession).apply(logicalPlan)
- }
-}
-
-
-/**
- * Session state implementation to override sql parser and adding strategies
- *
- * @param sparkSession
- */
-class CarbonSessionStateBuilder(sparkSession: SparkSession,
- parentState: Option[SessionState] = None)
- extends HiveSessionStateBuilder(sparkSession, parentState) {
-
- override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
-
- experimentalMethods.extraStrategies =
- Seq(new StreamingTableStrategy(sparkSession),
- new CarbonLateDecodeStrategy,
- new DDLStrategy(sparkSession)
- )
- experimentalMethods.extraOptimizations = Seq(new CarbonIUDRule,
- new CarbonUDFTransformRule,
- new CarbonLateDecodeRule)
-
- /**
- * Internal catalog for managing table and database states.
- */
- /**
- * Create a [[CarbonSessionCatalogBuild]].
- */
- override protected lazy val catalog: CarbonSessionCatalog = {
- val catalog = new CarbonSessionCatalog(
- externalCatalog,
- session.sharedState.globalTempViewManager,
- functionRegistry,
- sparkSession,
- conf,
- SessionState.newHadoopConf(session.sparkContext.hadoopConfiguration, conf),
- sqlParser,
- resourceLoader)
- parentState.foreach(_.catalog.copyStateTo(catalog))
- catalog
- }
-
- private def externalCatalog: HiveExternalCatalog =
- session.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog]
-
- /**
- * Create a Hive aware resource loader.
- */
- override protected lazy val resourceLoader: HiveSessionResourceLoader = {
- val client: HiveClient = externalCatalog.client.newSession()
- new HiveSessionResourceLoader(session, client)
- }
-
- override lazy val optimizer: Optimizer = new CarbonOptimizer(catalog, conf, experimentalMethods)
-
- override protected def analyzer: Analyzer = new CarbonAnalyzer(catalog, conf, sparkSession,
- new Analyzer(catalog, conf) {
-
- override val extendedResolutionRules: Seq[Rule[LogicalPlan]] =
- new ResolveHiveSerdeTable(session) +:
- new FindDataSourceTable(session) +:
- new ResolveSQLOnFile(session) +:
- new CarbonIUDAnalysisRule(sparkSession) +:
- new CarbonPreInsertionCasts(sparkSession) +: customResolutionRules
-
- override val extendedCheckRules: Seq[LogicalPlan => Unit] =
- PreWriteCheck :: HiveOnlyCheck :: Nil
-
- override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
- new DetermineTableStats(session) +:
- RelationConversions(conf, catalog) +:
- PreprocessTableCreation(session) +:
- PreprocessTableInsertion(conf) +:
- DataSourceAnalysis(conf) +:
- HiveAnalysis +:
- customPostHocResolutionRules
- }
- )
-
- override protected def newBuilder: NewBuilder = new CarbonSessionStateBuilder(_, _)
-
-}
-
-
-class CarbonOptimizer(
- catalog: SessionCatalog,
- conf: SQLConf,
- experimentalMethods: ExperimentalMethods)
- extends SparkOptimizer(catalog, conf, experimentalMethods) {
-
- override def execute(plan: LogicalPlan): LogicalPlan = {
- // In case scalar subquery add flag in relation to skip the decoder plan in optimizer rule, And
- // optimize whole plan at once.
- val transFormedPlan = plan.transform {
- case filter: Filter =>
- filter.transformExpressions {
- case s: ScalarSubquery =>
- val tPlan = s.plan.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- ScalarSubquery(tPlan, s.children, s.exprId)
- case e: Exists =>
- val tPlan = e.plan.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- Exists(tPlan, e.children.map(_.canonicalized), e.exprId)
-
- case In(value, Seq(l@ListQuery(sub, _, exprId))) =>
- val tPlan = sub.transform {
- case lr: LogicalRelation
- if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
- lr
- }
- In(value, Seq(ListQuery(tPlan, l.children , exprId)))
- }
- }
- super.execute(transFormedPlan)
- }
-}
-
-class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
- extends SparkSqlAstBuilder(conf) {
-
- val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
-
- override def visitCreateHiveTable(ctx: CreateHiveTableContext): LogicalPlan = {
- val fileStorage = helper.getFileStorage(ctx.createFileFormat)
-
- if (fileStorage.equalsIgnoreCase("'carbondata'") ||
- fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
- helper.createCarbonTable(
- tableHeader = ctx.createTableHeader,
- skewSpecContext = ctx.skewSpec,
- bucketSpecContext = ctx.bucketSpec,
- partitionColumns = ctx.partitionColumns,
- columns = ctx.columns,
- tablePropertyList = ctx.tablePropertyList,
- locationSpecContext = ctx.locationSpec(),
- tableComment = Option(ctx.STRING()).map(string),
- ctas = ctx.AS,
- query = ctx.query)
- } else {
- super.visitCreateHiveTable(ctx)
- }
- }
-
- override def visitChangeColumn(ctx: ChangeColumnContext): LogicalPlan = {
-
- val newColumn = visitColType(ctx.colType)
- if (!ctx.identifier.getText.equalsIgnoreCase(newColumn.name)) {
- throw new MalformedCarbonCommandException(
- "Column names provided are different. Both the column names should be same")
- }
-
- val (typeString, values) : (String, Option[List[(Int, Int)]]) = newColumn.dataType match {
- case d:DecimalType => ("decimal", Some(List((d.precision, d.scale))))
- case _ => (newColumn.dataType.typeName.toLowerCase, None)
- }
-
- val alterTableChangeDataTypeModel =
- AlterTableDataTypeChangeModel(new CarbonSpark2SqlParser().parseDataType(typeString, values),
- new CarbonSpark2SqlParser()
- .convertDbNameToLowerCase(Option(ctx.tableIdentifier().db).map(_.getText)),
- ctx.tableIdentifier().table.getText.toLowerCase,
- ctx.identifier.getText.toLowerCase,
- newColumn.name.toLowerCase)
-
- CarbonAlterTableDataTypeChangeCommand(alterTableChangeDataTypeModel)
- }
-
-
- override def visitAddTableColumns(ctx: AddTableColumnsContext): LogicalPlan = {
-
- val cols = Option(ctx.columns).toSeq.flatMap(visitColTypeList)
- val fields = parser.getFields(cols)
- val tblProperties = scala.collection.mutable.Map.empty[String, String]
- val tableModel = new CarbonSpark2SqlParser().prepareTableModel (false,
- new CarbonSpark2SqlParser().convertDbNameToLowerCase(Option(ctx.tableIdentifier().db)
- .map(_.getText)),
- ctx.tableIdentifier.table.getText.toLowerCase,
- fields,
- Seq.empty,
- tblProperties,
- None,
- true)
-
- val alterTableAddColumnsModel = AlterTableAddColumnsModel(
- Option(ctx.tableIdentifier().db).map(_.getText),
- ctx.tableIdentifier.table.getText,
- tblProperties.toMap,
- tableModel.dimCols,
- tableModel.msrCols,
- tableModel.highcardinalitydims.getOrElse(Seq.empty))
-
- CarbonAlterTableAddColumnCommand(alterTableAddColumnsModel)
- }
-
- override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = {
- super.visitCreateTable(ctx)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala b/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
deleted file mode 100644
index 2128ffd..0000000
--- a/integration/spark2/src/main/spark2.2/CarbonSqlConf.scala
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive
-
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.internal.SQLConf.buildConf
-
-import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
-import org.apache.carbondata.core.util.CarbonProperties
-
-/**
- * To initialize dynamic values default param
- */
-class CarbonSQLConf(sparkSession: SparkSession) {
-
- val carbonProperties = CarbonProperties.getInstance()
-
- /**
- * To initialize dynamic param defaults along with usage docs
- */
- def addDefaultCarbonParams(): Unit = {
- val ENABLE_UNSAFE_SORT =
- buildConf(CarbonCommonConstants.ENABLE_UNSAFE_SORT)
- .doc("To enable/ disable unsafe sort.")
- .booleanConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
- val CARBON_CUSTOM_BLOCK_DISTRIBUTION =
- buildConf(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION)
- .doc("To set carbon task distribution.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
- val BAD_RECORDS_LOGGER_ENABLE =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE)
- .doc("To enable/ disable carbon bad record logger.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants
- .CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
- val BAD_RECORDS_ACTION =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION)
- .doc("To configure the bad records action.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
- val IS_EMPTY_DATA_BAD_RECORD =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD)
- .doc("Property to decide weather empty data to be considered bad/ good record.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT
- .toBoolean)
- val SORT_SCOPE =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE)
- .doc("Property to specify sort scope.")
- .stringConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- val BATCH_SORT_SIZE_INMB =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB)
- .doc("Property to specify batch sort size in MB.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
- val SINGLE_PASS =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS)
- .doc("Property to enable/disable single_pass.")
- .booleanConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
- val BAD_RECORD_PATH =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH)
- .doc("Property to configure the bad record location.")
- .stringConf
- .createWithDefault(carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- val GLOBAL_SORT_PARTITIONS =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS)
- .doc("Property to configure the global sort partitions.")
- .stringConf
- .createWithDefault(carbonProperties
- .getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
- CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
- val DATEFORMAT =
- buildConf(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT)
- .doc("Property to configure data format for date type columns.")
- .stringConf
- .createWithDefault(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
- val CARBON_INPUT_SEGMENTS = buildConf(
- "carbon.input.segments.<database_name>.<table_name>")
- .doc("Property to configure the list of segments to query.").stringConf
- .createWithDefault(carbonProperties
- .getProperty("carbon.input.segments.<database_name>.<table_name>", "*"))
- }
- /**
- * to set the dynamic properties default values
- */
- def addDefaultCarbonSessionParams(): Unit = {
- sparkSession.conf.set(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- carbonProperties.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
- CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT).toBoolean)
- sparkSession.conf.set(CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION,
- carbonProperties
- .getProperty(CarbonCommonConstants.CARBON_TASK_DISTRIBUTION,
- CarbonCommonConstants.CARBON_TASK_DISTRIBUTION_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
- CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT.toBoolean)
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
- carbonProperties.getProperty(CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS,
- CarbonCommonConstants.LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT))
- sparkSession.conf.set(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d3f3989/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
new file mode 100644
index 0000000..3c151f0
--- /dev/null
+++ b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+
+import scala.collection.generic.SeqFactory
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
+import org.apache.spark.sql.catalyst.catalog._
+import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, BoundReference, Exists, Expression, In, InterpretedPredicate, ListQuery, ScalarSubquery}
+import org.apache.spark.sql.catalyst.optimizer.Optimizer
+import org.apache.spark.sql.catalyst.parser.ParserInterface
+import org.apache.spark.sql.catalyst.parser.ParserUtils.string
+import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{AddTableColumnsContext, ChangeColumnContext, CreateHiveTableContext, CreateTableContext}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.command._
+import org.apache.spark.sql.execution.command.schema.{CarbonAlterTableAddColumnCommand, CarbonAlterTableDataTypeChangeCommand}
+import org.apache.spark.sql.execution.datasources.{FindDataSourceTable, LogicalRelation, PreWriteCheck, ResolveSQLOnFile, _}
+import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
+import org.apache.spark.sql.execution.{SparkOptimizer, SparkSqlAstBuilder}
+import org.apache.spark.sql.hive.client.HiveClient
+import org.apache.spark.sql.internal.{SQLConf, SessionState}
+import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
+import org.apache.spark.sql.parser.{CarbonHelperSqlAstBuilder, CarbonSpark2SqlParser, CarbonSparkSqlParser}
+import org.apache.spark.sql.types.DecimalType
+import org.apache.spark.util.CarbonReflectionUtils
+
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.spark.util.CarbonScalaUtil
+
+/**
+ * This class will have carbon catalog and refresh the relation from cache if the carbontable in
+ * carbon catalog is not same as cached carbon relation's carbon table
+ *
+ * @param externalCatalog
+ * @param globalTempViewManager
+ * @param sparkSession
+ * @param functionResourceLoader
+ * @param functionRegistry
+ * @param conf
+ * @param hadoopConf
+ */
+class CarbonSessionCatalog(
+ externalCatalog: HiveExternalCatalog,
+ globalTempViewManager: GlobalTempViewManager,
+ functionRegistry: FunctionRegistry,
+ sparkSession: SparkSession,
+ conf: SQLConf,
+ hadoopConf: Configuration,
+ parser: ParserInterface,
+ functionResourceLoader: FunctionResourceLoader)
+ extends HiveSessionCatalog(
+ externalCatalog,
+ globalTempViewManager,
+ new HiveMetastoreCatalog(sparkSession),
+ functionRegistry,
+ conf,
+ hadoopConf,
+ parser,
+ functionResourceLoader
+ ) {
+
+ lazy val carbonEnv = {
+ val env = new CarbonEnv
+ env.init(sparkSession)
+ env
+ }
+
+ def getCarbonEnv() : CarbonEnv = {
+ carbonEnv
+ }
+
+ // Initialize all listeners to the Operation bus.
+ CarbonEnv.initListeners()
+
+
+
+
+ override def lookupRelation(name: TableIdentifier): LogicalPlan = {
+ val rtnRelation = super.lookupRelation(name)
+ var toRefreshRelation = false
+ rtnRelation match {
+ case SubqueryAlias(_,
+ LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
+ toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
+ case LogicalRelation(_: CarbonDatasourceHadoopRelation, _, _) =>
+ toRefreshRelation = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
+ case SubqueryAlias(_, relation) if
+ relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
+ relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.HiveTableRelation") ||
+ relation.getClass.getName.equals(
+ "org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation") =>
+ val catalogTable =
+ CarbonReflectionUtils.getFieldOfCatalogTable(
+ "tableMeta",
+ relation).asInstanceOf[CatalogTable]
+ toRefreshRelation =
+ CarbonEnv.refreshRelationFromCache(catalogTable.identifier)(sparkSession)
+ case _ =>
+ }
+
+ if (toRefreshRelation) {
+ super.lookupRelation(name)
+ } else {
+ rtnRelation
+ }
+ }
+
+ /**
+ * returns hive client from HiveExternalCatalog
+ *
+ * @return
+ */
+ def getClient(): org.apache.spark.sql.hive.client.HiveClient = {
+ sparkSession.asInstanceOf[CarbonSession].sharedState.externalCatalog
+ .asInstanceOf[HiveExternalCatalog].client
+ }
+
+ override def createPartitions(
+ tableName: TableIdentifier,
+ parts: Seq[CatalogTablePartition],
+ ignoreIfExists: Boolean): Unit = {
+ try {
+ val table = CarbonEnv.getCarbonTable(tableName)(sparkSession)
+ // Get the properties from thread local
+ val carbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo
+ if (carbonSessionInfo != null) {
+ val updatedParts = CarbonScalaUtil.updatePartitions(carbonSessionInfo, parts, table)
+ super.createPartitions(tableName, updatedParts, ignoreIfExists)
+ } else {
+ super.createPartitions(tableName, parts, ignoreIfExists)
+ }
+ } catch {
+ case e: Exception =>
+ super.createPartitions(tableName, parts, ignoreIfExists)
+ }
+ }
+
+ /**
+ * This is alternate way of getting partition information. It first fetches all partitions from
+ * hive and then apply filter instead of querying hive along with filters.
+ * @param partitionFilters
+ * @param sparkSession
+ * @param identifier
+ * @return
+ */
+ def getPartitionsAlternate(partitionFilters: Seq[Expression],
+ sparkSession: SparkSession,
+ identifier: TableIdentifier) = {
+ val allPartitions = sparkSession.sessionState.catalog.listPartitions(identifier)
+ ExternalCatalogUtils.prunePartitionsByFilter(
+ sparkSession.sessionState.catalog.getTableMetadata(identifier),
+ allPartitions,
+ partitionFilters,
+ sparkSession.sessionState.conf.sessionLocalTimeZone)
+ }
+}
+
+
+class CarbonAnalyzer(catalog: SessionCatalog,
+ conf: SQLConf,
+ sparkSession: SparkSession,
+ analyzer: Analyzer) extends Analyzer(catalog, conf) {
+ override def execute(plan: LogicalPlan): LogicalPlan = {
+ var logicalPlan = analyzer.execute(plan)
+ logicalPlan = CarbonPreAggregateDataLoadingRules(sparkSession).apply(logicalPlan)
+ CarbonPreAggregateQueryRules(sparkSession).apply(logicalPlan)
+ }
+}
+
+
+/**
+ * Session state implementation to override sql parser and adding strategies
+ *
+ * @param sparkSession
+ */
+class CarbonSessionStateBuilder(sparkSession: SparkSession,
+ parentState: Option[SessionState] = None)
+ extends HiveSessionStateBuilder(sparkSession, parentState) {
+
+ override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
+
+ experimentalMethods.extraStrategies =
+ Seq(new StreamingTableStrategy(sparkSession),
+ new CarbonLateDecodeStrategy,
+ new DDLStrategy(sparkSession)
+ )
+ experimentalMethods.extraOptimizations = Seq(new CarbonIUDRule,
+ new CarbonUDFTransformRule,
+ new CarbonLateDecodeRule)
+
+ /**
+ * Internal catalog for managing table and database states.
+ */
+ /**
+ * Create a [[CarbonSessionCatalogBuild]].
+ */
+ override protected lazy val catalog: CarbonSessionCatalog = {
+ val catalog = new CarbonSessionCatalog(
+ externalCatalog,
+ session.sharedState.globalTempViewManager,
+ functionRegistry,
+ sparkSession,
+ conf,
+ SessionState.newHadoopConf(session.sparkContext.hadoopConfiguration, conf),
+ sqlParser,
+ resourceLoader)
+ parentState.foreach(_.catalog.copyStateTo(catalog))
+ catalog
+ }
+
+ private def externalCatalog: HiveExternalCatalog =
+ session.sharedState.externalCatalog.asInstanceOf[HiveExternalCatalog]
+
+ /**
+ * Create a Hive aware resource loader.
+ */
+ override protected lazy val resourceLoader: HiveSessionResourceLoader = {
+ val client: HiveClient = externalCatalog.client.newSession()
+ new HiveSessionResourceLoader(session, client)
+ }
+
+ override lazy val optimizer: Optimizer = new CarbonOptimizer(catalog, conf, experimentalMethods)
+
+ override protected def analyzer: Analyzer = new CarbonAnalyzer(catalog, conf, sparkSession,
+ new Analyzer(catalog, conf) {
+
+ override val extendedResolutionRules: Seq[Rule[LogicalPlan]] =
+ new ResolveHiveSerdeTable(session) +:
+ new FindDataSourceTable(session) +:
+ new ResolveSQLOnFile(session) +:
+ new CarbonIUDAnalysisRule(sparkSession) +:
+ new CarbonPreInsertionCasts(sparkSession) +: customResolutionRules
+
+ override val extendedCheckRules: Seq[LogicalPlan => Unit] =
+ PreWriteCheck :: HiveOnlyCheck :: Nil
+
+ override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
+ new DetermineTableStats(session) +:
+ RelationConversions(conf, catalog) +:
+ PreprocessTableCreation(session) +:
+ PreprocessTableInsertion(conf) +:
+ DataSourceAnalysis(conf) +:
+ HiveAnalysis +:
+ customPostHocResolutionRules
+ }
+ )
+
+ override protected def newBuilder: NewBuilder = new CarbonSessionStateBuilder(_, _)
+
+}
+
+
+class CarbonOptimizer(
+ catalog: SessionCatalog,
+ conf: SQLConf,
+ experimentalMethods: ExperimentalMethods)
+ extends SparkOptimizer(catalog, conf, experimentalMethods) {
+
+ override def execute(plan: LogicalPlan): LogicalPlan = {
+ // In case scalar subquery add flag in relation to skip the decoder plan in optimizer rule, And
+ // optimize whole plan at once.
+ val transFormedPlan = plan.transform {
+ case filter: Filter =>
+ filter.transformExpressions {
+ case s: ScalarSubquery =>
+ val tPlan = s.plan.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ ScalarSubquery(tPlan, s.children, s.exprId)
+ case e: Exists =>
+ val tPlan = e.plan.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ Exists(tPlan, e.children.map(_.canonicalized), e.exprId)
+
+ case In(value, Seq(l@ListQuery(sub, _, exprId))) =>
+ val tPlan = sub.transform {
+ case lr: LogicalRelation
+ if lr.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
+ lr.relation.asInstanceOf[CarbonDatasourceHadoopRelation].isSubquery += true
+ lr
+ }
+ In(value, Seq(ListQuery(tPlan, l.children , exprId)))
+ }
+ }
+ super.execute(transFormedPlan)
+ }
+}
+
+class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSession: SparkSession)
+ extends SparkSqlAstBuilder(conf) {
+
+ val helper = new CarbonHelperSqlAstBuilder(conf, parser, sparkSession)
+
+ override def visitCreateHiveTable(ctx: CreateHiveTableContext): LogicalPlan = {
+ val fileStorage = helper.getFileStorage(ctx.createFileFormat)
+
+ if (fileStorage.equalsIgnoreCase("'carbondata'") ||
+ fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) {
+ helper.createCarbonTable(
+ tableHeader = ctx.createTableHeader,
+ skewSpecContext = ctx.skewSpec,
+ bucketSpecContext = ctx.bucketSpec,
+ partitionColumns = ctx.partitionColumns,
+ columns = ctx.columns,
+ tablePropertyList = ctx.tablePropertyList,
+ locationSpecContext = ctx.locationSpec(),
+ tableComment = Option(ctx.STRING()).map(string),
+ ctas = ctx.AS,
+ query = ctx.query)
+ } else {
+ super.visitCreateHiveTable(ctx)
+ }
+ }
+
+ override def visitChangeColumn(ctx: ChangeColumnContext): LogicalPlan = {
+
+ val newColumn = visitColType(ctx.colType)
+ if (!ctx.identifier.getText.equalsIgnoreCase(newColumn.name)) {
+ throw new MalformedCarbonCommandException(
+ "Column names provided are different. Both the column names should be same")
+ }
+
+ val (typeString, values) : (String, Option[List[(Int, Int)]]) = newColumn.dataType match {
+ case d:DecimalType => ("decimal", Some(List((d.precision, d.scale))))
+ case _ => (newColumn.dataType.typeName.toLowerCase, None)
+ }
+
+ val alterTableChangeDataTypeModel =
+ AlterTableDataTypeChangeModel(new CarbonSpark2SqlParser().parseDataType(typeString, values),
+ new CarbonSpark2SqlParser()
+ .convertDbNameToLowerCase(Option(ctx.tableIdentifier().db).map(_.getText)),
+ ctx.tableIdentifier().table.getText.toLowerCase,
+ ctx.identifier.getText.toLowerCase,
+ newColumn.name.toLowerCase)
+
+ CarbonAlterTableDataTypeChangeCommand(alterTableChangeDataTypeModel)
+ }
+
+
+ override def visitAddTableColumns(ctx: AddTableColumnsContext): LogicalPlan = {
+
+ val cols = Option(ctx.columns).toSeq.flatMap(visitColTypeList)
+ val fields = parser.getFields(cols)
+ val tblProperties = scala.collection.mutable.Map.empty[String, String]
+ val tableModel = new CarbonSpark2SqlParser().prepareTableModel (false,
+ new CarbonSpark2SqlParser().convertDbNameToLowerCase(Option(ctx.tableIdentifier().db)
+ .map(_.getText)),
+ ctx.tableIdentifier.table.getText.toLowerCase,
+ fields,
+ Seq.empty,
+ tblProperties,
+ None,
+ true)
+
+ val alterTableAddColumnsModel = AlterTableAddColumnsModel(
+ Option(ctx.tableIdentifier().db).map(_.getText),
+ ctx.tableIdentifier.table.getText,
+ tblProperties.toMap,
+ tableModel.dimCols,
+ tableModel.msrCols,
+ tableModel.highcardinalitydims.getOrElse(Seq.empty))
+
+ CarbonAlterTableAddColumnCommand(alterTableAddColumnsModel)
+ }
+
+ override def visitCreateTable(ctx: CreateTableContext): LogicalPlan = {
+ super.visitCreateTable(ctx)
+ }
+}